{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from __future__ import print_function\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.init as init\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#torch.nn: - Neural networks can be constructed using the torch.nn package."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Output size :  torch.Size([100, 5])\n",
      "Output size :  torch.Size([100, 5])\n"
     ]
    }
   ],
   "source": [
    "x = Variable(torch.randn(100, 10))\n",
    "y = Variable(torch.randn(100, 30))\n",
    "\n",
    "linear = nn.Linear(in_features=10, out_features=5, bias=True)\n",
    "output_linear = linear(x)\n",
    "print('Output size : ', output_linear.size())\n",
    "\n",
    "bilinear = nn.Bilinear(in1_features=10, in2_features=30, out_features=5, bias=True)\n",
    "output_bilinear = bilinear(x, y)\n",
    "print('Output size : ', output_bilinear.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Output size :  torch.Size([100, 10])\n",
      "Output size :  torch.Size([100, 30])\n"
     ]
    }
   ],
   "source": [
    "x = Variable(torch.randn(100, 10))\n",
    "y = Variable(torch.randn(100, 30))\n",
    "\n",
    "sig = nn.Sigmoid()\n",
    "output_sig = sig(x)\n",
    "output_sigy = sig(y)\n",
    "print('Output size : ', output_sig.size())\n",
    "print('Output size : ', output_sigy.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([ 1.5325, -0.1192,  0.7443,  0.4696, -0.3996, -0.2290, -0.4939,  0.7307,\n",
      "         1.9625,  1.8852])\n",
      "tensor([0.8224, 0.4702, 0.6779, 0.6153, 0.4014, 0.4430, 0.3790, 0.6750, 0.8768,\n",
      "        0.8682])\n"
     ]
    }
   ],
   "source": [
    "print(x[0])\n",
    "print(output_sig[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Output size :  torch.Size([100, 10])\n",
      "Output size :  torch.Size([100, 30])\n"
     ]
    }
   ],
   "source": [
    "x = Variable(torch.randn(100, 10))\n",
    "y = Variable(torch.randn(100, 30))\n",
    "\n",
    "func = nn.Tanh()\n",
    "output_x = func(x)\n",
    "output_y = func(y)\n",
    "print('Output size : ', output_x.size())\n",
    "print('Output size : ', output_y.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([ 1.9040,  0.9865,  0.5902, -0.0119, -1.1839,  1.1157,  0.0491, -0.5634,\n",
      "        -0.5739, -0.0915])\n",
      "tensor([ 0.9566,  0.7559,  0.5300, -0.0119, -0.8287,  0.8061,  0.0490, -0.5105,\n",
      "        -0.5182, -0.0913])\n",
      "tensor([-0.6040,  0.4982, -0.3855,  0.8471,  0.1878, -0.9120, -0.1233,  0.4404,\n",
      "        -1.5006, -0.7271,  1.2325, -0.9579, -2.1563, -0.1659,  0.3000, -1.0285,\n",
      "        -2.0542,  1.0215,  0.8815,  2.7387, -1.5575,  0.9263, -0.2420,  1.4894,\n",
      "         0.4274,  0.7546,  0.9662,  0.9938,  0.3938,  0.5279])\n",
      "tensor([-0.5399,  0.4607, -0.3674,  0.6896,  0.1856, -0.7221, -0.1227,  0.4140,\n",
      "        -0.9053, -0.6213,  0.8433, -0.7433, -0.9736, -0.1644,  0.2913, -0.7733,\n",
      "        -0.9677,  0.7705,  0.7072,  0.9917, -0.9150,  0.7289, -0.2374,  0.9032,\n",
      "         0.4031,  0.6379,  0.7470,  0.7590,  0.3747,  0.4838])\n"
     ]
    }
   ],
   "source": [
    "print(x[0])\n",
    "print(output_x[0])\n",
    "print(y[0])\n",
    "print(output_y[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Output size :  torch.Size([100, 10])\n",
      "Output size :  torch.Size([100, 30])\n"
     ]
    }
   ],
   "source": [
    "x = Variable(torch.randn(100, 10))\n",
    "y = Variable(torch.randn(100, 30))\n",
    "\n",
    "func = nn.LogSigmoid()\n",
    "output_x = func(x)\n",
    "output_y = func(y)\n",
    "print('Output size : ', output_x.size())\n",
    "print('Output size : ', output_y.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([-0.1882,  2.1937,  0.2143, -0.5117,  0.2683, -1.5367, -0.1725, -0.1069,\n",
      "        -0.1077, -0.8164])\n",
      "tensor([-0.7917, -0.1057, -0.5917, -0.9814, -0.5680, -1.7315, -0.7831, -0.7480,\n",
      "        -0.7485, -1.1825])\n",
      "tensor([-0.6322,  1.1316,  0.6100, -1.0292, -0.6858, -0.1626,  0.5313,  0.3809,\n",
      "         0.7807,  1.2960,  0.4634, -0.1030,  0.7024,  0.1865, -0.6282,  1.0392,\n",
      "         0.0105, -1.7282, -1.1551, -1.5041,  0.3256,  1.0605,  0.0639, -0.7072,\n",
      "        -1.2789, -2.3255,  0.9516, -0.5806, -0.2420, -0.2606])\n",
      "tensor([-1.0584, -0.2795, -0.4340, -1.3347, -1.0937, -0.7777, -0.4624, -0.5207,\n",
      "        -0.3771, -0.2419, -0.4880, -0.7460, -0.4024, -0.6042, -1.0558, -0.3029,\n",
      "        -0.6879, -1.8917, -1.4290, -1.7048, -0.5435, -0.2974, -0.6617, -1.1080,\n",
      "        -1.5245, -2.4188, -0.3265, -1.0250, -0.8215, -0.8319])\n"
     ]
    }
   ],
   "source": [
    "print(x[0])\n",
    "print(output_x[0])\n",
    "print(y[0])\n",
    "print(output_y[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Output size :  torch.Size([100, 10])\n",
      "Output size :  torch.Size([100, 30])\n"
     ]
    }
   ],
   "source": [
    "x = Variable(torch.randn(100, 10))\n",
    "y = Variable(torch.randn(100, 30))\n",
    "\n",
    "func = nn.ReLU()\n",
    "output_x = func(x)\n",
    "output_y = func(y)\n",
    "print('Output size : ', output_x.size())\n",
    "print('Output size : ', output_y.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([-0.7700, -1.1810,  0.7188,  0.1945,  0.3566, -0.3520, -2.2136,  0.3014,\n",
      "         0.3616, -0.3331])\n",
      "tensor([0.0000, 0.0000, 0.7188, 0.1945, 0.3566, 0.0000, 0.0000, 0.3014, 0.3616,\n",
      "        0.0000])\n",
      "tensor([-0.0335, -0.3917, -0.0635,  0.7571,  0.9160, -0.5633, -1.4463,  1.6797,\n",
      "         1.7013, -0.9248, -0.6231,  1.1278, -0.4329,  0.6814,  0.2152,  0.4618,\n",
      "         0.2440,  1.2931, -0.4524, -0.7411, -0.7834,  1.0311,  1.6846, -0.8909,\n",
      "         1.0070,  0.5172,  0.4322,  0.0947,  0.7641,  0.3708])\n",
      "tensor([0.0000, 0.0000, 0.0000, 0.7571, 0.9160, 0.0000, 0.0000, 1.6797, 1.7013,\n",
      "        0.0000, 0.0000, 1.1278, 0.0000, 0.6814, 0.2152, 0.4618, 0.2440, 1.2931,\n",
      "        0.0000, 0.0000, 0.0000, 1.0311, 1.6846, 0.0000, 1.0070, 0.5172, 0.4322,\n",
      "        0.0947, 0.7641, 0.3708])\n"
     ]
    }
   ],
   "source": [
    "print(x[0])\n",
    "print(output_x[0])\n",
    "print(y[0])\n",
    "print(output_y[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Output size :  torch.Size([100, 10])\n",
      "Output size :  torch.Size([100, 30])\n"
     ]
    }
   ],
   "source": [
    "x = Variable(torch.randn(100, 10))\n",
    "y = Variable(torch.randn(100, 30))\n",
    "\n",
    "func = nn.LeakyReLU()\n",
    "output_x = func(x)\n",
    "output_y = func(y)\n",
    "print('Output size : ', output_x.size())\n",
    "print('Output size : ', output_y.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([ 0.4568,  1.6528, -1.8593,  0.2683,  1.0176, -0.5545,  0.1327,  0.1778,\n",
      "        -1.6823, -0.4703])\n",
      "tensor([ 0.4568,  1.6528, -0.0186,  0.2683,  1.0176, -0.0055,  0.1327,  0.1778,\n",
      "        -0.0168, -0.0047])\n",
      "tensor([ 2.3048, -0.0535,  0.3125, -0.1602, -1.7538,  0.3496,  0.0635,  1.3486,\n",
      "        -1.0511,  2.0831, -0.6695, -0.7135, -1.3160, -0.5777,  1.8069, -0.4711,\n",
      "        -0.4981, -1.2562, -0.6824, -1.8397,  1.3567, -1.5145, -0.1898,  1.3763,\n",
      "        -0.6095, -0.4215,  0.7330, -1.6545,  0.4332,  0.5325])\n",
      "tensor([ 2.3048, -0.0005,  0.3125, -0.0016, -0.0175,  0.3496,  0.0635,  1.3486,\n",
      "        -0.0105,  2.0831, -0.0067, -0.0071, -0.0132, -0.0058,  1.8069, -0.0047,\n",
      "        -0.0050, -0.0126, -0.0068, -0.0184,  1.3567, -0.0151, -0.0019,  1.3763,\n",
      "        -0.0061, -0.0042,  0.7330, -0.0165,  0.4332,  0.5325])\n"
     ]
    }
   ],
   "source": [
    "print(x[0])\n",
    "print(output_x[0])\n",
    "print(y[0])\n",
    "print(output_y[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = torch.linspace(-10, 10, 1500)  \n",
    "x = Variable(x)\n",
    "x_1 = x.data.numpy()   # tranforming into numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_relu = F.relu(x).data.numpy()\n",
    "y_sigmoid = torch.sigmoid(x).data.numpy()\n",
    "y_tanh = torch.tanh(x).data.numpy()\n",
    "y_softplus = F.softplus(x).data.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x1d7b4a48fd0>"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAawAAAD8CAYAAAArMZDvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAHUVJREFUeJzt3XmYVNWdxvHvT0AWRYGmcVA0jYa4\nEQVsVGRUdhDZFBIxiohmMDquxKiMM2p09HFEgxJxQURcUJBNFlFBQH1MIgQQUVkCKsRGkGZXEdnO\n/HEK0zbd0HTdrlO36v08Tz9UV92qevv2pd6+t26dY845RERE0t0hoQOIiIiUhQpLRERiQYUlIiKx\noMISEZFYUGGJiEgsqLBERCQWVFgiIhILKiwREYkFFZaIiMRC5VQ+Wd26dV1eXl4qn1JERNLY/Pnz\n1zvncsuybEoLKy8vj3nz5qXyKUVEJI2Z2aqyLqtDgiIiEgsqLBERiQUVloiIxEJK38Mqyc6dOyko\nKGD79u2ho6RMtWrVaNCgAVWqVAkdRUQkNoIXVkFBATVr1iQvLw8zCx2nwjnn2LBhAwUFBTRs2DB0\nHBGR2Ah+SHD79u3k5ORkRVkBmBk5OTlZtUcpIhKF4IUFZE1Z7ZVtP6+ISBTSorBEREQORIUFVKpU\niSZNmtC4cWO6du3K5s2bD3ifww8/fJ/rrrzySsaNG3fA5URE5OCpsIDq1auzcOFCPvnkE+rUqcPQ\noUNDRxIRkWJUWMW0aNGC1atX//j9oEGDaN68Oaeddhp33313wGQiItkt+GntRd18MyxcGO1jNmkC\njz5atmV3797NzJkzufrqqwGYPn06y5cvZ+7cuTjn6NatG++99x7nnXdetCFFROSAtIcFfP/99zRp\n0oScnBw2btxI+/btAV9Y06dPp2nTpjRr1oylS5eyfPnyUh+npLP/dEagiEg0DriHZWYjgC7AOudc\n48R1dYAxQB6wEvi1c25TsmHKuicUtb3vYW3ZsoUuXbowdOhQbrzxRpxzDBw4kGuuuaZMj5OTk8Om\nTf9aDRs3bqRu3boVFVtEJKuUZQ9rJNCp2HV3ADOdc42AmYnvY+/II49kyJAhPPzww+zcuZOOHTsy\nYsQIvv32WwBWr17NunXrSr1/q1atGDNmDDt27ABg5MiRtG7dOiXZRUQy3QH3sJxz75lZXrGruwOt\nEpefB94Bbo8wVzBNmzbl9NNPZ/To0fTp04clS5bQokULwJ+i/tJLL1GvXj22bdtGgwYNfrzfgAED\nGDBgAPPnz+eMM86gUqVKnHDCCTz11FOhfhQRkQozaRJ07AjVqqXuOc05d+CFfGFNLXJIcLNzrlaR\n2zc552qXct/+QH+A44477oxVq346V9eSJUs4+eSTy5s/trL15xaR+HvqKbj2Wvjf/4U770zuscxs\nvnMuvyzLVvhJF865Yc65fOdcfm5umWZBFhGRNDV8uC+rrl3hD39I7XOXt7C+NrP6AIl/S39jR0RE\nMsLzz0P//tCpE4wdC4cemtrnL29hTQb6Ji73BSYlE6IshyUzSbb9vCISf6NGQb9+0LYtTJgAVaum\nPsMBC8vMXgH+BpxoZgVmdjXwINDezJYD7RPfl0u1atXYsGFD1ryI750Pq1oq36kUEUnCmDFwxRXQ\nqpU/2aJ69TA5ynKW4KWl3NQ2igANGjSgoKCAwsLCKB4uFvbOOCwiku7Gj4fLLoOWLWHKFKhRI1yW\n4EMzValSRTPvioikocmToXdvOOsseP11OOywsHk0NJOIiOxj2jTo1QuaNYM33oCaNUMnUmGJiEgx\n06fDxRfDL38Jb70FRxwROpGnwhIRkR/NnAndu8NJJ8GMGVCr1oHvkyoqLBERAeDdd/0Hgn/+c3j7\nbahTJ3Sin1JhiYgI778PF14IeXl+LysdJ5pQYYmIZLkPPoDOneGYY3xZ1asXOlHJVFgiIlls3jw/\n6nq9ejBrFtSvHzpR6VRYIiJZ6sMPoX17yMmB2bP9HlY6U2GJiGShRYugXTt/yvqsWXDssaETHZgK\nS0Qky3z6qR/Etnp1X1Z5eaETlY0KS0Qkiyxd6suqShV/GPCEE0InKjsVlohIlli+HNq08ZdnzYJG\njcLmOVjBB78VEZGK9/nnvqx27fJ7ViedFDrRwVNhiYhkuJUroXVr2LbNl9Wpp4ZOVD4qLBGRDPbl\nl37PautW/6Hg004Lnaj8VFgiIhlq9Wq/Z7Vhgx8bsFmz0ImSo8ISEclAa9b4Pat16/x0Ic2bh06U\nPBWWiEiGWbfOn7q+erWfz+rss0MnioYKS0Qkg6xf70ewWLnSzxTcsmXoRNFRYYmIZIiNG/3YgMuX\nw9SpcP75oRNFS4UlIpIBNm+GDh1g8WKYPNkfEsw0KiwRkZjbssVPEbJoEbz2mr+ciVRYIiIx9s03\ncMEFsGABjB/vJ2LMVCosEZGY+u47P6393LkwZgx06xY6UcVSYYmIxNC2bdC1K/zlL/Dyy9CzZ+hE\nFU+FJSISM9u3Q48e8M478OKLcMkloROlRlLTi5jZLWb2qZl9YmavmFm1qIKJiMi+fvgBLrrID7U0\nYgRcdlnoRKlT7sIys2OAG4F851xjoBLQO6pgIiLyUzt2QK9e8OabMGwYXHll6ESplewEjpWB6mZW\nGagBfJV8JBERKW7nTujd238g+Ikn4Le/DZ0o9cpdWM651cDDwD+BNcAW59z0qIKJiIi3a5c/9Ddx\nIgwZAtdeGzpRGMkcEqwNdAcaAkcDh5nZ5SUs19/M5pnZvMLCwvInFRHJQrt3wxVXwNix8MgjcMMN\noROFk8whwXbAF865QufcTmACcE7xhZxzw5xz+c65/Nzc3CSeTkQku+zeDVddBa+8Ag8+CAMGhE4U\nVjKF9U/gbDOrYWYGtAWWRBNLRCS77dkD/fvDCy/AvffC7beHThReMu9hzQHGAQuAjxOPNSyiXCIi\nWcs5uO46f9r6//yP/5IkPzjsnLsbuDuiLCIiWc85/z7V00/DHXfAH/8YOlH6SPa0dhERiYhz/n2q\noUPh97+HBx4As9Cp0ocKS0QkDTjn36d69FG46SYYNEhlVZwKS0QkMOf8+1SDBvnPWA0erLIqiQpL\nRCSwe++F++/3o1c8/rjKqjQqLBGRgB54AO65x48L+PTTcIhelUulVSMiEsigQXDnnXD55TB8uMrq\nQLR6REQCGDwYbrvND2j73HNQqVLoROlPhSUikmJDh/rT13v29BMwVtZUumWiwhIRSaFhw+D666F7\ndz9GoMqq7FRYIiIpMmIEXHMNdO4MY8ZAlSqhE8WLCktEJAVeeMGftt6hA4wfD1Wrhk4UPyosEZEK\n9vLL0K8ftGkDr70G1aqFThRPKiwRkQo0diz06QPnnguTJ0P16qETxZcKS0SkgkycCJdeCuecA1On\nQo0aoRPFmwpLRKQCTJ0Kl1wCzZvDtGlw+OGhE8WfCktEJGJvvuk/Y3X66f5yzZqhE2UGFZaISIRm\nzIAePeDUU2H6dDjyyNCJMocKS0QkIrNnQ7ducOKJvrhq1w6dKLOosEREIvDee9ClC5xwArz9NuTk\nhE6UeVRYIiJJ+utf/egVxx0HM2dCbm7oRJlJhSUikoS5c6FTJzj6aJg1C446KnSizKXCEhEpp/nz\n/VBLubm+rOrXD50os6mwRETKYeFCaN8eatXyZdWgQehEmU+FJSJykD7+GNq18x8Gnj0bfvaz0Imy\ngwpLROQgLF4Mbdv60dZnzYKGDUMnyh4qLBGRMlq2zI+4XqmS37P6+c9DJ8oumutSRKQMVqzwZbVn\nD7z7LvziF6ETZZ+k9rDMrJaZjTOzpWa2xMxaRBVMRCRdfPGFL6sffvCHAU8+OXSi7JTsHtZjwJvO\nuV5mdiigwfNFJKOsWgWtW8O33/qyatw4dKLsVe7CMrMjgPOAKwGcczuAHdHEEhEJr6DA71lt3uxH\nsGjSJHSi7JbMIcHjgULgOTP70MyGm9lhEeUSEQnqq6/8ntX69X7U9TPOCJ1IkimsykAz4EnnXFPg\nO+CO4guZWX8zm2dm8woLC5N4OhGR1Fi71u9ZrV3r57M688zQiQSSK6wCoMA5Nyfx/Th8gf2Ec26Y\ncy7fOZefqxEhRSTNFRb6z1l9+aWfKbiFTiVLG+UuLOfcWuBLMzsxcVVbYHEkqUREAtiwwY9g8cUX\nfor7c88NnUiKSvYswRuAUYkzBD8H+iUfSUQk9TZt8mMDLlvmy6p169CJpLikCss5txDIjyiLiEgQ\nmzf7Udc//RQmTfJ7WZJ+NNKFiGS1rVv9fFYffQQTJvjLkp5UWCKStb791s8UPH8+jB3rp7iX9KXC\nEpGs9N13cOGF8MEHMHo09OgROpEciApLRLLO999Dt27w/vvw0kvQq1foRFIWKiwRySrbt/u9qdmz\n4fnn4dJLQyeSslJhiUjW+OEH6NnTD7X07LPQp0/oRHIwNIGjiGSFHTvg17/2o1c8/TRcdVXoRHKw\nVFgikvF27vSH/iZPhscfh/79QyeS8lBhiUhG27XLH/qbMAEGD4b//M/QiaS8VFgikrF274Yrr4Qx\nY+Chh+Dmm0MnkmSosEQkI+3ZA7/9LYwaBfffD3/4Q+hEkiwVlohknD174JprYORIuOce+K//Cp1I\noqDCEpGM4hxcfz0MHw533gl33RU6kURFhSUiGcM5uOkmePJJuO02uO8+MAudSqKiwhKRjOAc3Hor\n/PnPcMst8OCDKqtMo8ISkdhzzr9P9ac/+cOBjzyisspEKiwRib277/Z7VNdcA0OGqKwylQpLRGLt\nvvv819VXwxNPqKwymQpLRGLrwQf9WYBXXAHDhsEhekXLaPr1ikgsPfIIDBwIv/kNjBihssoG+hWL\nSOwMGeLPCPzVr/ycVpUqhU4kqaDCEpFYefJJ/1mriy7ywy5V1qx+WUOFJSKx8cwzcN110LUrjB4N\nVaqETiSppMISkVgYOdKftn7BBTB2LBx6aOhEkmoqLBFJey+95GcIbtfOz2tVtWroRBKCCktE0tro\n0dC3L7RqBa+9BtWqhU4koaiwRCRtjRsHl18OLVvClClQo0boRBKSCktE0tKkSXDppXDWWfD663DY\nYaETSWhJF5aZVTKzD81sahSBRERef91/xqpZM3jjDahZM3QiSQdR7GHdBCyJ4HFERHjrLbj4Yjjt\nNH/5iCNCJ5J0kVRhmVkD4EJgeDRxRCSbzZwJPXrAKafA9OlQq1boRJJOkt3DehS4DdhT2gJm1t/M\n5pnZvMLCwiSfTkQy1Tvv+A8EN2oEM2ZAnTqhE0m6KXdhmVkXYJ1zbv7+lnPODXPO5Tvn8nNzc8v7\ndCKSwd5/H7p0gbw8ePttqFs3dCJJR8nsYbUEupnZSmA00MbMXooklYhkjb/9zY9eccwx/pBgvXqh\nE0m6KndhOecGOucaOOfygN7ALOfc5ZElE5GM9/e/Q6dOcNRRMGsW1K8fOpGkM30OS0SCWLAAOnSA\nnByYPdvvYYnsTyQD8zvn3gHeieKxRCTzffQRtG/vT1mfNQuOPTZ0IokD7WGJSEp98okfxLZGDb9n\nlZcXOpHEhQpLRFJmyRJo29bPYzVrFhx/fOhEEicqLBFJiX/8A9q0ATNfVo0ahU4kcaPJpUWkwn32\nmS+r3bv9YcCTTgqdSOJIhSUiFWrlSl9W33/vy+rUU0MnkrhSYYlIhfnnP6F1a9i61R8GPO200Ikk\nzlRYIlIhVq/2e1abNvnhlpo2DZ1I4k6FJSKRW7PG71mtW+cHss3PD51IMoEKS0Qi9fXXfs/qq6/8\nfFZnnRU6kWQKFZaIRGb9ev+h4FWr/EzBLVuGTiSZRIUlIpHYuNGX1YoVfor7888PnUgyjQpLRJK2\naZMfG3DpUpg82R8SFImaCktEkrJlC3Ts6McInDjRj8AuUhFUWCJSbt984ydf/PBDGD8eOncOnUgy\nmQpLRMrlu+98Qc2dC2PGQLduoRNJplNhichB27YNunSBv/4VXn4ZevYMnUiygQpLRA7K999D9+7w\n7rvw4otwySWhE0m20PQiIlJm27fDxRfDzJnw3HNw2WWhE0k20R6WiJTJjh3Qqxe8+SYMHw59+4ZO\nJNlGe1gickA7d/pDf6+/Dk8+CVdfHTqRZCMVlojs165d8JvfwGuvwZAh8LvfhU4k2UqFJSKl2r0b\n+vSBcePgkUfghhtCJ5JspsISkRLt3g39+sHo0fDggzBgQOhEku1UWCKyjz174D/+w5+2ft99cPvt\noROJqLBEpJg9e+Daa/1p63fdBf/936ETiXgqLBH5kXP+faphw2DgQLjnntCJRP5FhSUigC+rW26B\nJ56AW2+F++8Hs9CpRP6l3IVlZsea2WwzW2Jmn5rZTVEGE5HUcQ5uuw0eewxuugkeekhlJeknmZEu\ndgG/d84tMLOawHwzm+GcWxxRNhFJAef8+1QPPwzXXQeDB6usJD2Vew/LObfGObcgcfkbYAlwTFTB\nRCQ1/vhHeOABf1bgn/+sspL0Fcl7WGaWBzQF5pRwW38zm2dm8woLC6N4OhGJyP33+8Lq1w+eegoO\n0bvaksaS3jzN7HBgPHCzc25r8dudc8Occ/nOufzc3Nxkn05EIvLQQ/5QYJ8+8MwzKitJf0ltomZW\nBV9Wo5xzE6KJJCIVbfBg/2Hg3r39560qVQqdSOTAkjlL0IBngSXOuT9FF0lEKtLjj/thlnr29CNZ\nqKwkLpLZw2oJ9AHamNnCxFfniHKJSAV4+mn/weDu3eGVV6CyZsSTGCn35uqcex/Q+UQiMfHss35q\nkAsvhDFjoEqV0IlEDo7eZhXJAs8/709b79jRTxVStWroRCIHT4UlkuFeftmftt62LUycCNWqhU4k\nUj4qLJEM9uqr/rT188+HSZOgevXQiUTKT4UlkqEmTvRT259zDkyZAjVqhE4kkhwVlkgGmjIFLrkE\nmjeHadPg8MNDJxJJngpLJMO88Qb06gVNmsCbb0LNmqETiURDhSWSQaZPh4sugsaN4a234MgjQycS\niY4KSyRDzJrlPxB80km+uGrXDp1IJFoqLJEM8N570LUrnHACzJgBOTmhE4lET4UlEnN/+Qt07gzH\nHQczZ4ImRZBMpcISibE5c+CCC+Doo/0hwaOOCp1IpOKosERiat48P9RSbq4vq/r1QycSqVgqLJEY\n+vBD6NDBn1gxezY0aBA6kUjFU2GJxMzHH0P79v7DwLNn+/euRLKBCkskRhYv9oPYVq3qyyovL3Qi\nkdRRYYnExLJl0KaNnyF49mx/CrtINtF8oyIxsGKFLyvnfFn94hehE4mkngpLJM19/jm0bg07dviy\nOvnk0IlEwlBhiaSxVat8WW3b5k9db9w4dCKRcFRYImnqyy99WW3d6kewOP300IlEwlJhiaSh1av9\ne1YbNsDbb0OzZqETiYSnwhJJM2vX+rJau9aPut68eehEIulBhSWSRtat85+zKijwky+2aBE6kUj6\nUGGJpIn166FdO/jiCz+t/bnnhk4kkl5UWCJpYONGP9zS8uUwZQq0ahU6kUj6UWGJBLZ5sx/IdvFi\nmDzZ72WJyL5UWCIBbd0KnTrBokUwYYKfLkRESpbUWIJm1snMlpnZCjO7I6pQItngm2/85Ivz58Or\nr0KXLqETiaS3cheWmVUChgIXAKcAl5rZKVEFE8lk333nC2rOHHjlFejRI3QikfSXzCHBM4EVzrnP\nAcxsNNAdWBxFsJLs2uUPoYjE2bZt0LcvvP8+jBoFvXqFTiQSD8kU1jHAl0W+LwDOSi7O/i1dCr/8\nZUU+g0hqHHIIjBwJvXuHTiISH8kUlpVwndtnIbP+QH+A45KcGrV+fXjssaQeQiQtnHWW/xKRskum\nsAqAY4t83wD4qvhCzrlhwDCA/Pz8fQrtYOTkwI03JvMIIiISV8mcJfh3oJGZNTSzQ4HewORoYomI\niPxUufewnHO7zOx64C2gEjDCOfdpZMlERESKSOqDw865acC0iLKIiIiUKqkPDouIiKSKCktERGJB\nhSUiIrGgwhIRkVhQYYmISCyosEREJBZUWCIiEgsqLBERiQUVloiIxIIKS0REYkGFJSIisaDCEhGR\nWFBhiYhILJhzSc2peHBPZlYIrIrgoeoC6yN4nFRR3ooXt8xxywvxy6y8FS+KzD9zzuWWZcGUFlZU\nzGyecy4/dI6yUt6KF7fMccsL8cusvBUv1Zl1SFBERGJBhSUiIrEQ18IaFjrAQVLeihe3zHHLC/HL\nrLwVL6WZY/keloiIZJ+47mGJiEiWScvCMrNfmdmnZrbHzPKL3TbQzFaY2TIz61jK/Rua2RwzW25m\nY8zs0NQk//H5x5jZwsTXSjNbWMpyK83s48Ry81KZsViOe8xsdZHMnUtZrlNiva8wsztSnbNYlkFm\nttTMFpnZRDOrVcpyQdfxgdaZmVVNbC8rEttsXqozFslyrJnNNrMlif9/N5WwTCsz21JkW7krRNZi\nmfb7OzZvSGIdLzKzZiFyJrKcWGTdLTSzrWZ2c7Flgq9jMxthZuvM7JMi19UxsxmJ19UZZla7lPv2\nTSyz3Mz6RhrMOZd2X8DJwInAO0B+ketPAT4CqgINgc+ASiXc/1Wgd+LyU8C1AX+WR4C7SrltJVA3\nDdb3PcCtB1imUmJ9Hw8cmvg9nBIwcwegcuLy/wH/l27ruCzrDLgOeCpxuTcwJuA6rQ80S1yuCfyj\nhLytgKmhMpbndwx0Bt4ADDgbmBM6c5HtYy3+c0hptY6B84BmwCdFrnsIuCNx+Y6S/s8BdYDPE//W\nTlyuHVWutNzDcs4tcc4tK+Gm7sBo59wPzrkvgBXAmUUXMDMD2gDjElc9D/SoyLylSWT5NfBKiOeP\n2JnACufc5865HcBo/O8jCOfcdOfcrsS3HwANQmXZj7Kss+74bRT8Nts2sd2knHNujXNuQeLyN8AS\n4JgQWSLWHXjBeR8AtcysfuhQQFvgM+dcFIMpRMo59x6wsdjVRbfV0l5XOwIznHMbnXObgBlAp6hy\npWVh7ccxwJdFvi9g3/9QOcDmIi9mJS2TKucCXzvnlpdyuwOmm9l8M+ufwlwluT5xuGREKbv6ZVn3\noVyF/wu6JCHXcVnW2Y/LJLbZLfhtOKjEocmmwJwSbm5hZh+Z2RtmdmpKg5XsQL/jdN12e1P6H7Pp\nto4BjnLOrQH/xw1Qr4RlKnRdV47qgQ6Wmb0N/FsJN93pnJtU2t1KuK74aY5lWSZpZcx/Kfvfu2rp\nnPvKzOoBM8xsaeIvm8jtLy/wJHAffj3dhz+MeVXxhyjhvhV6imlZ1rGZ3QnsAkaV8jApW8clSJvt\n9WCY2eHAeOBm59zWYjcvwB/C+jbxXudrQKNUZyzmQL/jdFzHhwLdgIEl3JyO67isKnRdByss51y7\nctytADi2yPcNgK+KLbMev8tfOfEXa0nLJO1A+c2sMnAxcMZ+HuOrxL/rzGwi/hBShbyYlnV9m9kz\nwNQSbirLuo9UGdZxX6AL0NYlDqCX8BgpW8clKMs627tMQWKbOZJ9D8WkjJlVwZfVKOfchOK3Fy0w\n59w0M3vCzOo654KNgVeG33HKt90yuABY4Jz7uvgN6biOE742s/rOuTWJQ6rrSlimAP8e3F4N8Oci\nRCJuhwQnA70TZ1Y1xP/VMbfoAokXrtlAr8RVfYHS9tgqUjtgqXOuoKQbzewwM6u59zL+JIJPSlq2\nohU7nn9RKTn+DjQyfwbmofjDGZNTka8kZtYJuB3o5pzbVsoyoddxWdbZZPw2Cn6bnVVa+Va0xHtn\nzwJLnHN/KmWZf9v7HpuZnYl/DdmQupT75CnL73gycEXibMGzgS17D20FVOrRl3Rbx0UU3VZLe119\nC+hgZrUTby10SFwXjZBnopT2hX/RLAB+AL4G3ipy2534M6+WARcUuX4acHTi8vH4IlsBjAWqBvgZ\nRgK/K3bd0cC0Ihk/Snx9ij/MFWp9vwh8DCzCb5T1i+dNfN8Zf+bYZyHzJrKswB8rX5j42numXVqt\n45LWGXAvvmgBqiW20RWJbfb4gOv03/GHbxYVWa+dgd/t3ZaB6xPr8iP8yS7nBN4OSvwdF8tswNDE\n7+Bjipx5HChzDXwBHVnkurRax/gyXQPsTLwWX41/b3UmsDzxb53EsvnA8CL3vSqxPa8A+kWZSyNd\niIhILMTtkKCIiGQpFZaIiMSCCktERGJBhSUiIrGgwhIRkVhQYYmISCyosEREJBZUWCIiEgv/D8Gh\n6q/7Uxy7AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1d7b47f9278>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(7, 4))\n",
    "plt.plot(x_1, y_relu, c='blue', label='ReLU')\n",
    "plt.ylim((-1, 11))\n",
    "plt.legend(loc='best')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x1d7b47f9780>"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbcAAAD8CAYAAAD0f+rwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XmYFNXZ9/HvDQwg+64IsqhoZBGX\nCfhEIyCoiBE0cY1GERU3kqCiYGISxceFqIlvIqKIS1zCEn0ko6IoLoAalQFxAURHRRlBBUTEKMLA\n/f5RPdAMPQtMTVd39e9zXXV1LWfq3FPT03efqlOnzN0RERGJk1pRByAiIhI2JTcREYkdJTcREYkd\nJTcREYkdJTcREYkdJTcREYmdUJKbmd1nZl+a2bvlbD/TzN5OTK+aWc8w6hUREUklrJbbA8DACrZ/\nDPRx9wOB64GJIdUrIiKygzph7MTd55hZpwq2v5q0+BrQPox6RUREUgklue2k84CnU20ws+HAcICG\nDRse+qMf/SidcYmISIabP3/+andvXVm5tCY3M+tHkNyOSLXd3SeSOGWZn5/vhYWFaYxOREQynZl9\nUpVyaUtuZnYgMAk4zt3XpKteERHJPWm5FcDMOgD/B/zK3d9PR50iIpK7Qmm5mdlkoC/QysyKgT8B\neQDufhfwR6AlcKeZAZS4e34YdYuIiJQVVm/JMyrZfj5wfhh1iYhks02bNlFcXMyGDRuiDiWj1a9f\nn/bt25OXl7dLPx9Fb0kRkZxVXFxM48aN6dSpE4kzWVKGu7NmzRqKi4vp3LnzLu1Dw2+JiKTRhg0b\naNmypRJbBcyMli1bVqt1q+QmIpJmSmyVq+4xUnITEZHYUXITEclx559/PosXL67ROgYNGsTXX3+9\nw/prr72WW2+9NfT61KFERCTHTZo0qcbrmDFjRo3XkUwtNxGRHPLf//6X448/np49e9K9e3emTp1K\n3759KR3u8N5772W//fajb9++XHDBBYwYMQKAoUOHcvHFF9OvXz/23ntvZs+ezbBhwzjggAMYOnTo\n1v1PnjyZHj160L17d0aPHr11fadOnVi9ejUAN9xwA/vvvz8DBgxg6dKlNfJ7quUmIhKRkSNh4cJw\n93nQQXD77eVvf+aZZ9hzzz156qmnAFi3bh0TJkwAYMWKFVx//fUsWLCAxo0bc9RRR9Gz57bHb65d\nu5YXXniBgoICTjjhBF555RUmTZrEj3/8YxYuXEibNm0YPXo08+fPp3nz5hxzzDFMnz6dE088ces+\n5s+fz5QpU3jzzTcpKSnhkEMO4dBDDw33IKCWm4hITunRowezZs1i9OjRzJ07l6ZNm27d9sYbb9Cn\nTx9atGhBXl4ep5xyynY/e8IJJ2Bm9OjRg913350ePXpQq1YtunXrxrJly5g3bx59+/aldevW1KlT\nhzPPPJM5c+Zst4+5c+dy0kkn0aBBA5o0acLgwYNr5PdUy01EJCIVtbBqyn777cf8+fOZMWMGV199\nNcccc8zWbe5e4c/Wq1cPgFq1am2dL10uKSmhTp2qpZR03AqhlpuISA5ZsWIFDRo04KyzzmLUqFEs\nWLBg67ZevXoxe/Zs1q5dS0lJCY899thO7bt3797Mnj2b1atXs3nzZiZPnkyfPn22K3PkkUfy+OOP\n8/3337N+/XqeeOKJUH6vstRyExHJIe+88w5XXnkltWrVIi8vjwkTJjBq1CgA2rVrx+9+9zt69+7N\nnnvuSdeuXbc7bVmZtm3bctNNN9GvXz/cnUGDBjFkyJDtyhxyyCGcdtppHHTQQXTs2JGf/vSnof5+\npayyZmhU9LBSEYmjJUuWcMABB0QdRrm+/fZbGjVqRElJCSeddBLDhg3jpJNOiiSWVMfKzOZX5aky\nOi0pIiJbXXvttRx00EF0796dzp07b9fTMZvotKSIiGxVE6OFREEtNxGRNMvUy0GZpLrHSMlNRCSN\n6tevz5o1a5TgKlD6PLf69evv8j50WlJEJI3at29PcXExq1atijqUjFb6JO5dpeQmIpJGeXl5u/x0\naak6nZYUEZHYUXITEZHYUXITEZHYCSW5mdl9Zvalmb1bznYzs7+ZWZGZvW1mh4RRr4iISCphtdwe\nAAZWsP04oEtiGg5MCKleERGRHYTSW9Ld55hZpwqKDAEe9ODGjtfMrJmZtXX3lWHULyKSadxh8+bt\np5KSqq/bsmXbfsKcKttnqt+jouWdWTd4MFTxqTjVlq5bAdoBy5OWixPrtktuZjacoGVHhw4d0hSa\niOSazZth7VpYswa++grWrYP16+Hbb3ecStdv2AA//JB62rhxx+XS5CTbfPtt/JJbqifT7ZDX3X0i\nMBGCpwLUdFAiEi/r1sHy5fDZZztOX365LZl9/XXqlkWy2rWhcWNo1AgaNoTddoN69YKpcWNo1SqY\nr1t32/rSqW7d4EO8du3tp6quK51Kn+lpFu5U2T7LKruuKmVSravGgCM7LV3JrRjYK2m5PbAiTXWL\nSIxs3gzvvw9vvx28fvABFBUFr6tX71i+dWto1w7atIF99oGWLbdNLVoEr02bbktkjRoF83Xrpv7A\nluyQruRWAIwwsylAb2CdrreJSGXc4b334OWXYcECePPNIKl9//22Mu3bQ5cu8POfw777QseOQTJr\n1w7atg1aUpJ7QkluZjYZ6Au0MrNi4E9AHoC73wXMAAYBRcB3wLlh1Csi8eIOS5fCM8/AnDkwd+62\n1lizZnDQQXDhhXDwwXDggbDfftCgQbQxS2YKq7fkGZVsd+DSMOoSkXjZvBlmz4aCAnjySfjww2D9\n3nvD8cfDT38aTF266DShVJ0GThaRtHOHt96Chx+Gf/4TVq4MTh/27w9XXAGDBgWnF0V2lZKbiKTN\nhg0weTL8/e/B9bO8vCCRnXlm8NqwYdQRSlwouYlIjVuzBm6/He66K7iG1q0bjB8Pp58e9FgUCZuS\nm4jUmDVr4C9/gb/9Df77XxgyBH7zG+jbV9fPpGYpuYlI6DZuhDvugOuuC0b4OPVU+MMfghabSDoo\nuYlIqGbOhN/+NujSP3Ag3HILdO8edVSSa/Q8NxEJxdq1cM45QULbvBmeeAJmzFBik2io5SYi1fb0\n03D++fDFF3DNNcGkkUEkSmq5icguKymBMWOCbvzNm8Prr8P11yuxSfTUchORXbJyZdCVf86cYEis\n229P76jvIhVRchORnbZwYTA01tdfw4MPwq9+FXVEIttTchORnfLMM3DKKcFAxq++Cj17Rh2RyI50\nzU1Equz+++FnPwseLfPaa0pskrmU3ESkSu6+G4YNCwY3njMneF6aSKZSchORSt1xB1x0UdBqKygI\nnlQtksmU3ESkQhMmwK9/DSeeCI89pm7+kh3UoUREyjVtGlx6KZxwQjCflxd1RCJVo5abiKT0/PNw\n1llw+OEwdaoSm2QXJTcR2cFbbwWnIfffP7jGtttuUUcksnOU3ERkO6tWBc9da9o0uKetefOoIxLZ\nebrmJiJbbdoU3KD9+ecwd666+0v2UnITka1GjoTZs+Ghh+DHP446GpFdF8ppSTMbaGZLzazIzMak\n2N7BzF40szfN7G0zGxRGvSISnkcegTvvhCuuCDqSiGSzaic3M6sNjAeOA7oCZ5hZ1zLFrgGmufvB\nwOnAndWtV0TC88EHwU3ahx8ON98cdTQi1RdGy60XUOTuH7n7RmAKMKRMGQeaJOabAitCqFdEQvDD\nD8Gja/Ly4J//hDq6WCExEMbbuB2wPGm5GOhdpsy1wLNm9mugITAg1Y7MbDgwHKBDhw4hhCYilRkz\nBhYsgOnTQf92EhdhtNwsxTovs3wG8IC7twcGAQ+Z2Q51u/tEd8939/zWrVuHEJqIVOSFF4KHjI4Y\nEXT/F4mLMJJbMbBX0nJ7djzteB4wDcDd/wPUB1qFULeI7KL164NR/rt0gXHjoo5GJFxhJLd5QBcz\n62xmdQk6jBSUKfMp0B/AzA4gSG6rQqhbRHbRVVfBp5/CAw9AgwZRRyMSrmonN3cvAUYAM4ElBL0i\nF5nZWDMbnCh2BXCBmb0FTAaGunvZU5cikiazZsFdd8Hll8NPfhJ1NCLhs0zNMfn5+V5YWBh1GCKx\n89130K1b8OiaN9/UuJGSXcxsvrvnV1ZOnX5FcswNN8CyZcFIJEpsElcaOFkkhyxZArfcAmefDUce\nGXU0IjVHyU0kR7jDJZdAo0ZBghOJM52WFMkRjzwCL70UdCRp0ybqaERqllpuIjlg/XoYNQp69YIL\nLog6GpGap5abSA4YNw6++CJ4qnYtfaWVHKC3uUjMffop3HYbnHlm0HITyQVKbiIxd/XVweuNN0Yb\nh0g6KbmJxNjrrwePsbniCo34L7lFyU0kptyD4bV23x1Gj446GpH0UocSkZiaPh1efRUmToTGjaOO\nRiS91HITiaHNm+Gaa2D//eHcc6OORiT91HITiaHJk2HxYpg6Ferov1xykFpuIjGzaRP86U/Qsyec\nfHLU0YhEQ9/pRGLm/vvho4/gySd1w7bkLr31RWJkwwYYOxb+539g0KCooxGJjlpuIjEyYQJ89hk8\n9BCYRR2NSHTUchOJie++g5tuggEDoF+/qKMRiZaSm0hM3HcfrFoVdCYRyXVKbiIxsGkT3HorHH44\nHHFE1NGIRE/X3ERiYOpU+OQTuOOOqCMRyQxquYlkuS1b4OaboXt39ZAUKRVKcjOzgWa21MyKzGxM\nOWVONbPFZrbIzP4ZRr0iAjNmwKJFweDIuq9NJFDt05JmVhsYDxwNFAPzzKzA3RcnlekCXA0c7u5r\nzaxNdesVkcDNN0PHjnDaaVFHIpI5wvie1wsocveP3H0jMAUYUqbMBcB4d18L4O5fhlCvSM57+WV4\n5RUYNQry8qKORiRzhJHc2gHLk5aLE+uS7QfsZ2avmNlrZjYwhHpFct64cdCqFQwbFnUkIpkljN6S\nqcZB8BT1dAH6Au2BuWbW3d2/3m5HZsOB4QAd9NhgkQq9804wfuTYsdCgQdTRiGSWMFpuxcBeScvt\ngRUpyvzb3Te5+8fAUoJktx13n+ju+e6e37p16xBCE4mvP/8ZGjaESy+NOhKRzBNGcpsHdDGzzmZW\nFzgdKChTZjrQD8DMWhGcpvwohLpFctKyZcEz2y68EFq0iDoakcxT7eTm7iXACGAmsASY5u6LzGys\nmQ1OFJsJrDGzxcCLwJXuvqa6dYvkqttuC7r9X3ZZ1JGIZCZzL3t5LDPk5+d7YWFh1GGIZJxVq4Ku\n/2ecAffeG3U0IullZvPdPb+ycrrlUyTL/P3vwXPbrrwy6khEMpeSm0gWWb8+GD/ypJPgRz+KOhqR\nzKXkJpJF7rkH1q4NhtoSkfIpuYlkiR9+CDqS9OsHvXpFHY1IZtMjb0SyxCOPwIoVcP/9UUcikvnU\nchPJAps3BzdtH3wwHH101NGIZD613ESywL//DUuXBg8ltVQD3onIdtRyE8lw7sFjbfbZB37xi6ij\nEckOarmJZLiXXoJ58+Duu6F27aijEckOarmJZLibb4bdd4ezz446EpHsoeQmksEWLIBnnw3GkKxf\nP+poRLKHkptIBhs3Dpo0gYsuijoSkeyi5CaSoT74AB59FC65BJo2jToakeyi5CaSoW69FfLy4Le/\njToSkeyj5CaSgVauhAcegHPPhT32iDoakeyj5CaSgW6/HUpKYNSoqCMRyU5KbiIZZt06uOsuOPXU\n4MZtEdl5Sm4iGWbCBPjmG7jqqqgjEcleSm4iGeT774NTksceGwySLCK7RslNJIP84x/wxRcwZkzU\nkYhkNyU3kQxRUhI81qZ3b+jTJ+poRLKbBk4WyRBTp8LHHwenJfVYG5HqCaXlZmYDzWypmRWZWbkn\nVMzsZDNzM8sPo16RuNiyJRgguVs3+NnPoo5GJPtVu+VmZrWB8cDRQDEwz8wK3H1xmXKNgd8Ar1e3\nTpG4efJJePddePhhqKWLBSLVFsa/US+gyN0/cveNwBRgSIpy1wN/BjaEUKdIbLjDjTdC585w2mlR\nRyMSD2Ekt3bA8qTl4sS6rczsYGAvd38yhPpEYuWll+D114P72uroKrhIKMJIbqkuffvWjWa1gL8C\nV1S6I7PhZlZoZoWrVq0KITSRzHfjjcH4kUOHRh2JSHyEkdyKgb2SltsDK5KWGwPdgZfMbBlwGFCQ\nqlOJu09093x3z2/dunUIoYlktnnzYNYsuPxyPYxUJExhJLd5QBcz62xmdYHTgYLSje6+zt1buXsn\nd+8EvAYMdvfCEOoWyWo33QTNmulhpCJhq3Zyc/cSYAQwE1gCTHP3RWY21swGV3f/InH17rvw+OPw\n619D48ZRRyMSL+bulZeKQH5+vhcWqnEn8XXqqfDMM7BsGbRoEXU0ItnBzOa7e6X3SuuOGpEIvPMO\n/OtfwVO2ldhEwqfkJhKB666DJk3gssuijkQknpTcRNLsrbfgscfUahOpSUpuIml23XXQtKlabSI1\nSclNJI0WLgx6SI4cCc2bRx2NSHwpuYmkUWmrbeTIqCMRiTclN5E0ee01mD4drrgiuHFbRGqOkptI\nGrjD6NHQpo2utYmkg8YgF0mDp5+GOXNg/Hho1CjqaETiTy03kRq2eTOMGQP77gsXXBB1NCK5QS03\nkRr2yCPBiCRTp0JeXtTRiOQGtdxEatCGDfCHP0B+Ppx8ctTRiOQOtdxEatBf/wqffgr33Qe19FVS\nJG307yZSQz77DG64AYYMgf79o45GJLcouYnUkDFjYNMmuO22qCMRyT1KbiI14D//gYcfDm7Y3mef\nqKMRyT1KbiIh27IFfvMb2HNP+N3voo5GJDepQ4lIyCZNgsJCeOgh3bAtEhW13ERCtHIlXHUV9O0L\nZ54ZdTQiuUvJTSREI0cG97bdfTeYRR2NSO5SchMJyVNPwbRpcM01sN9+UUcjktuU3ERC8O23cMkl\n0LVrcFpSRKKlDiUiIbj8cli+HObOhbp1o45GREJpuZnZQDNbamZFZjYmxfbLzWyxmb1tZs+bWccw\n6hXJBE89BffcA1deCYcfHnU0IgIhJDczqw2MB44DugJnmFnXMsXeBPLd/UDgUeDP1a1XJBOsXg3n\nnQcHHghjx0YdjYiUCqPl1gsocveP3H0jMAUYklzA3V909+8Si68B7UOoVyRS7nDRRfDVV8E9bfXq\nRR2RiJQKI7m1A5YnLRcn1pXnPODpVBvMbLiZFZpZ4apVq0IITaTmTJgAjz0G//u/QctNRDJHGMkt\n1d08nrKg2VlAPnBLqu3uPtHd8909v3Xr1iGEJlIzCgvhsstg0CAYNSrqaESkrDB6SxYDeyUttwdW\nlC1kZgOA3wN93P2HEOoVicTatXDKKbDHHvDgg3pOm0gmCiO5zQO6mFln4DPgdOCXyQXM7GDgbmCg\nu38ZQp0ikdi8Gc4+O3hW29y50LJl1BGJSCrV/s7p7iXACGAmsASY5u6LzGysmQ1OFLsFaAT8y8wW\nmllBdesVicLVV8OTT8Ltt0Pv3lFHIyLlCeUmbnefAcwos+6PSfMDwqhHJEqTJsEtt8CIEcFoJCKS\nuXS1QKQKXnwRLr4Yjj0W/vrXqKMRkcoouYlUorAQhgwJBkOeOhXqaNA6kYyn5CZSgXffDVprLVvC\ns89C06ZRRyQiVaHkJlKOoiI45phg5JFZs6BdRUMTiEhGUXITSWHRIjjySNi4EZ57DvbZJ+qIRGRn\nKLmJlDFvXpDYAF56Cbp1izQcEdkFSm4iSWbNgv79oUmT4Cbt7t2jjkhEdoWSm0jC+PEwcCB07Agv\nv6xTkSLZTMlNct7GjcFN2SNGwHHHwauvqvOISLZTcpOc9uGHcMQRweNrrrwSpk+Hxo2jjkpEqku3\no0pOcocpU+DCC6F2bXj0UfjFL6KOSkTCopab5JzPPoOf/xx++Uvo0QMWLlRiE4kbJTfJGSUlwenH\nrl3hmWdg3DiYPTvoQCIi8aLTkhJ77vD008E1tcWL4aij4O67Yd99o45MRGqKWm4SW+7BaP4DBsDx\nxwe9Ih97LLiXTYlNJN6U3CR2tmyBggL4yU+CVtqiRcHDRRctCq61mUUdoYjUNJ2WlNhYuRLuvx/u\nuQeWLYNOneDOO2HoUNhtt4iDE5G0UnKTrPbNN/DEEzBtGsyYEXQa6dcPbrop6AGZlxd1hCISBSU3\nyTqffBKM1F9QADNnBtfS2rWDkSPhgguCh4qKSG5TcpOM5g7Ll8Prrwfd9p97Dt5/P9i2115w6aVw\n8slw2GFQS1eQRSRByU0yxpYt8PHHQXf9d96BN94IktrnnwfbGzaEPn3g4ovh6KOD+9XUOUREUlFy\nk7TauDFoiS1bFpxeXLYMPvooSGjvvQfff7+tbJcuQTf+ww6D3r3hwAOhbt2oIheRbBJKcjOzgcD/\nA2oDk9z95jLb6wEPAocCa4DT3H1ZGHVLdLZsgfXrg04dpdPXX8OXX6aePv886NHovm0ftWoF18u6\ndg06gnTtGjwc9IADoGnT6H43Eclu1U5uZlYbGA8cDRQD88yswN0XJxU7D1jr7vua2enAOOC06tZd\nmZKS4APYfdsHakXzlW3PhLIQ/E4lJbB5866/ls7/8ANs2LDj9P33qdclJ7P16ys+/vXrw+67Q5s2\n0LYt9OwJHToEXfQ7dgxe27dXj0YRCV8YLbdeQJG7fwRgZlOAIUBychsCXJuYfxS4w8zMPfk7fPjG\njIHbbqvJGuKlbt0gIZWddtsteG3WLEhOTZoEU9Om2+ZLp2bNgmTWpk1wjUzXxEQkCmEkt3bA8qTl\nYqB3eWXcvcTM1gEtgdXJhcxsODAcoEOHDtUO7LjjoEWL0n1v+6CtaD6qsjuzLzOoUyeYatfecX5n\nXuvVCxJXvXrqbSgi8RFGckv13bxsi6wqZXD3icBEgPz8/Gq36vr3DyYREcktYXxXLwb2SlpuD6wo\nr4yZ1QGaAl+FULeIiMgOwkhu84AuZtbZzOoCpwMFZcoUAOck5k8GXqjp620iIpK7qn1aMnENbQQw\nk+BWgPvcfZGZjQUK3b0AuBd4yMyKCFpsp1e3XhERkfKEcp+bu88AZpRZ98ek+Q3AKWHUJSIiUhn1\njxMRkdhRchMRkdhRchMRkdhRchMRkdhRchMRkdhRchMRkdhRchMRkdhRchMRkdhRchMRkdhRchMR\nkdhRchMRkdhRchMRkdhRchMRkdhRchMRkdhRchMRkdhRchMRkdhRchMRkdhRchMRkdhRchMRkdhR\nchMRkdhRchMRkdhRchMRkdipVnIzsxZm9pyZfZB4bZ6izEFm9h8zW2Rmb5vZadWpU0REpDLVbbmN\nAZ539y7A84nlsr4Dznb3bsBA4HYza1bNekVERMpV3eQ2BPhHYv4fwIllC7j7++7+QWJ+BfAl0Lqa\n9YqIiJSrTjV/fnd3Xwng7ivNrE1Fhc2sF1AX+LCc7cOB4YnFb81saTXjA2gFrA5hP+mUbTEr3pqX\nbTFnW7yQfTHnarwdq1LI3L3iAmazgD1SbPo98A93b5ZUdq2773DdLbGtLfAScI67v1aV4MJgZoXu\nnp+u+sKQbTEr3pqXbTFnW7yQfTEr3opV2nJz9wHlbTOzL8ysbaLV1pbglGOqck2Ap4Br0pnYREQk\nN1X3mlsBcE5i/hzg32ULmFld4HHgQXf/VzXrExERqVR1k9vNwNFm9gFwdGIZM8s3s0mJMqcCRwJD\nzWxhYjqomvXujIlprCss2Raz4q152RZztsUL2Rez4q1ApdfcREREso1GKBERkdhRchMRkdiJRXIz\ns1MSw3ttMbP8MtuuNrMiM1tqZseW8/Odzez1xDBiUxOdYNImUWfp9chlZrawnHLLzOydRLnCdMZY\nJo5rzeyzpJgHlVNuYOK4F5lZqtFr0sLMbjGz9xLDvz1e3gg5UR/fyo6XmdVLvFeKEu/XTumOsUw8\ne5nZi2a2JPH/99sUZfqa2bqk98ofo4g1KZ4K/8YW+FviGL9tZodEEWdSPPsnHbuFZvaNmY0sUybS\nY2xm95nZl2b2btK6SodmTJQ7J1HmAzM7J1WZXebuWT8BBwD7E9xHl5+0vivwFlAP6Exw83jtFD8/\nDTg9MX8XcHGEv8ttwB/L2bYMaJUBx/taYFQlZWonjvfeBDfuvwV0jSjeY4A6iflxwLhMO75VOV7A\nJcBdifnTgakRvw/aAock5hsD76eIuS/wZJRx7szfGBgEPA0YcBjwetQxl3mPfA50zKRjTNBh8BDg\n3aR1fwbGJObHpPqfA1oAHyVemyfmm4cVVyxabu6+xN1TjWYyBJji7j+4+8dAEdAruYCZGXAU8Ghi\nVcphxNIhEcupwOQo6g9ZL6DI3T9y943AFIK/R9q5+7PuXpJYfA1oH0UclajK8Uoe7u5RoH/iPRMJ\nd1/p7gsS8+uBJUC7qOIJyRCC25bcg3tymyXu4c0E/YEP3f2TqANJ5u5zgK/KrK50aEbgWOA5d//K\n3dcCzxGMPxyKWCS3CrQDlictF7PjP19L4OukD79UZdLlp8AXnhiLMwUHnjWz+YmhyqI0InHa5r5y\nTjlU5dhHYRjBN/NUojy+VTleW8sk3q/rCN6/kUucIj0YeD3F5v8xs7fM7Gkz65bWwHZU2d84U9+3\nELTWy/vim0nHGMoMzQikGpqxRo91dceWTBurYBgwd9/h5vHSH0uxruy9D1UpU21VjP8MKm61He7u\nKywYw/M5M3sv8a0pdBXFC0wAric4TtcTnEodVnYXKX62xu47qcrxNbPfAyXAI+XsJm3HN4WMea/u\nLDNrBDwGjHT3b8psXkBwGu3bxLXZ6UCXdMeYpLK/caYe47rAYODqFJsz7RhXVY0e66xJbl7BMGAV\nKAb2SlpuD6woU2Y1wamHOolvw6nKVFtl8ZtZHeDnwKEV7GNF4vVLM3uc4FRWjXz4VvV4m9k9wJMp\nNlXl2IemCsf3HOBnQH9PnPBPsY+0Hd8UqnK8SssUJ94vTdnxdFBamVkeQWJ7xN3/r+z25GTn7jPM\n7E4za+XukQz4W4W/cVrftzvhOGCBu39RdkOmHeOEqgzNWExwvbBUe4J+E6GI+2nJAuD0RC+zzgTf\nZt5ILpD4oHsRODmxKuUwYmkwAHjP3YtTbTSzhmbWuHSeoJPEu6nK1rQy1yBOKieOeUAXC3qi1iU4\npVKQjvjKMrOBwGhgsLt/V05PpM3BAAABZElEQVSZqI9vVY5X8nB3JwMvlJeo0yFxve9eYIm7/6Wc\nMnuUXhe04KkgtYA16Ytyu1iq8jcuAM5O9Jo8DFhXenotYuWe1cmkY5yk0qEZgZnAMWbWPHFp45jE\nunBE1cMmzIngA7YY+AH4ApiZtO33BL3QlgLHJa2fAeyZmN+bIOkVAf8C6kXwOzwAXFRm3Z7AjKQY\n30pMiwhOt0V1vB8C3gHeJngTty0bb2J5EEEPug8jjreI4Nz+wsRU2uMwo45vquMFjCVIygD1E+/P\nosT7de+ojmkiniMITiO9nXRsBwEXlb6XgRGJ4/kWQWeen0QYb8q/cZl4DRif+Bu8Q1Lv6wjjbkCQ\nrJomrcuYY0yQdFcCmxKfw+cRXAt+Hvgg8doiUTYfmJT0s8MS7+ci4Nww49LwWyIiEjtxPy0pIiI5\nSMlNRERiR8lNRERiR8lNRERiR8lNRERiR8lNRERiR8lNRERi5/8DF0aExUDRlPYAAAAASUVORK5C\nYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1d7b4a66240>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(7, 4))\n",
    "plt.plot(x_1, y_sigmoid, c='blue', label='sigmoid')\n",
    "plt.ylim((-0.2, 1.2))\n",
    "plt.legend(loc='best')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x1d7b4b1eef0>"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbcAAAD8CAYAAAD0f+rwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAG5ZJREFUeJzt3XuUHHWd9/H3N5MbaMCQCwRCTEDu\noAGGyE0XCJfAytUF466Kyy4cL+zzeNsV5Rx0dc9Z1KOu64IQhIVlEVgVHiJmuQqLLAEygYQEgyQg\nyBDMVcIlCclkfs8f1YPD0JOZSfd0dVe/X+f0qaqu33R9p6ZmPlNVv6qKlBKSJBXJkLwLkCSp2gw3\nSVLhGG6SpMIx3CRJhWO4SZIKx3CTJBWO4SZJKhzDTZJUOIabJKlwhuZdQG/Gjh2bJk+enHcZkqQ6\nMn/+/NUppXF9tavbcJs8eTJtbW15lyFJqiMR8Xx/2nlYUpJUOIabJKlwDDdJUuHU7Tm3cjZv3kx7\nezsbN27Mu5RBNXLkSCZOnMiwYcPyLkWSGlJDhVt7ezujRo1i8uTJRETe5QyKlBJr1qyhvb2dKVOm\n5F2OJDWkhjosuXHjRsaMGVPYYAOICMaMGVP4vVNJGkwNFW5AoYOtSzN8j5I0mBou3CRJ6ovhNkAv\nv/wyl19++TZ//THHHOPF6ZI0yAy3Aao03CRJg89wG6CLLrqIZ555hqlTp/L5z3+e6dOnc8ghh3DQ\nQQdx2223AfDcc8+x3377cf7553PAAQdw4oknsmHDhjc/46c//SnTpk1j77335te//nVe34okFVZD\nXQrQ3ec+BwsWVPczp06Ff/mXrbe59NJLWbx4MQsWLKCjo4P169ezww47sHr1ag4//HBOO+00AJYu\nXcqNN97IVVddxTnnnMPPf/5zPvaxjwHQ0dHBo48+ypw5c/jHf/xH7rnnnup+I5LU5Bo23OpBSomv\nfvWrPPDAAwwZMoQXX3yRFStWADBlyhSmTp0KwKGHHspzzz335tedddZZZd+XJFVHVcItIq4BPgSs\nTCkdWGZ+AD8ATgHWA59MKT1WyTL72sOqhRtuuIFVq1Yxf/58hg0bxuTJk9+8Pm3EiBFvtmtpaXnL\nYcmueS0tLXR0dNS2aElqAtU653YtMGMr808G9iq9LgB+VKXl1tyoUaN49dVXAVi3bh3jx49n2LBh\n3HfffTz/fL+exCBJGmRV2XNLKT0QEZO30uR04D9SSgl4OCLeFRETUkovVWP5tTRmzBiOOuooDjzw\nQA477DCeeuopWltbmTp1Kvvuu2/e5UkNbfNm2LAhG3Z0wJYt2atrvOdwyxbo7ISUsq8vN9zavIG0\nUeWOPRaG1uhkWK3Oue0GvNBtur303lvCLSIuINuzY9KkSTUqbeB+8pOf9Nlm8eLFb45/6UtfenP8\n/vvvf3N87NixnnNT4XR2wsqV8PvfZ6/2dlizBtau/dNw7Vp4/fUsyDZsgPXrs+GWLXlXr8H06qvw\nznfWZlm1Crdy95N62/9CKaVZwCyA1tZW/1eS6twbb8Ajj8Cjj8KiRfDEE/DUU9Dz1qhDhsDo0bDT\nTtlr3DiYMgW23x622y57dR8fPhxaWrLX0KFvHfYcb2nJltF117pyw63NG0gbVWa77Wq3rFqFWzuw\ne7fpicDyGi1bUhU9/zz87Gdw++0wd24WcAC77goHHQTHHQd77AGTJmWviROzYBviVbWqoVqF22zg\nwoi4CXg/sG5bz7ellAp/Y+HkAX7VmU2bskC77DJ46KHsvfe9Dz7zGTjmGDjySBg7NtcSpbeo1qUA\nNwLHAGMjoh34GjAMIKV0BTCH7DKAZWSXAvz1tixn5MiRrFmzptCPvel6ntvIkSPzLkVi82a48kr4\n53+G5cth772z8bPPhj33zLs6qXfV6i350T7mJ+CzlS5n4sSJtLe3s2rVqko/qq51PYlbytPdd8OF\nF8LTT8MHPwhXXQUzZnh4UY2hoe5QMmzYMJ9OLQ2y9evh7/8eLr8821ObPRs+9CE7VKixNFS4SRpc\n7e1w2mnZfVu/8AX4p3+qbQ83qVoMN0lA1o3/pJOy689+8Qv48z/PuyJp2xlukli0KOvCv912WW/I\nA992h1ipsRhuUpNbuhSmT4cRI+C+++A978m7Iqly9nuSmtjLL8Opp2a3zDLYVCTuuUlNqrMT/uqv\n4Jln4N57s56RUlEYblKT+uEPYc6c7K4jH/xg3tVI1eVhSakJLV4MX/5ydkjy05/Ouxqp+gw3qcl0\ndsL558MOO8CPf+zF2SomD0tKTebaa+Hhh+G662D8+LyrkQaHe25SE/njH7PDkUcfDR//eN7VSIPH\ncJOayHe+kz0N+4c/9HCkis1wk5rEihXwgx/AzJkwdWre1UiDy3CTmsSll2ZPzf761/OuRBp8hpvU\nBFavhiuuyM6zebG2moHhJjWBH/0INm7MntMmNQPDTSq4jRvh3/4NTj4Z9t8/72qk2jDcpIL7yU9g\n5Ur44hfzrkSqHcNNKrgrroADDsie1yY1C8NNKrCFC2HevOx2W17XpmZiuEkFdvXV2UNIvRuJmo3h\nJhXUhg1w/fVw1lmw0055VyPVluEmFdQvf5k9afu88/KuRKo9w00qqJtugp13hmOPzbsSqfYMN6mA\nXn0123M7+2xoacm7Gqn2DDepgGbPzi7e/shH8q5EyofhJhXQzTfDxIlw5JF5VyLlw3CTCmbdOrjj\nDjjnHBjib7ialJu+VDB33AGbN2eXAEjNynCTCub222HMGDj88LwrkfJjuEkF0tEBc+bAKafYS1LN\nzXCTCmTuXFi7Fk49Ne9KpHwZblKB3H47DB0KJ52UdyVSvgw3qUB+8Qv4sz+DHXbIuxIpX4abVBC/\n/z0sWZKdb5OaXVXCLSJmRMRvI2JZRFxUZv4nI2JVRCwovf62GsuV9Cf33psNTzgh3zqkejC00g+I\niBbgMuAEoB2YFxGzU0q/6dH05pTShZUuT1J5994L48fDgQfmXYmUv2rsuU0DlqWUnk0pbQJuAk6v\nwudK6qeU4J57YPp0n7gtQXXCbTfghW7T7aX3evpwRDwRET+LiN2rsFxJJU8+CStWwPHH512JVB+q\nEW7l/k9MPaZ/AUxOKb0XuAe4ruwHRVwQEW0R0bZq1aoqlCY1h67zbYablKlGuLUD3ffEJgLLuzdI\nKa1JKb1RmrwKOLTcB6WUZqWUWlNKrePGjatCaVJzuOce2GsvmDQp70qk+lCNcJsH7BURUyJiODAT\nmN29QURM6DZ5GrCkCsuVRHaT5P/5n+x8m6RMxb0lU0odEXEhcCfQAlyTUnoyIr4BtKWUZgP/JyJO\nAzqAtcAnK12upMzjj2dP3j7uuLwrkepHxeEGkFKaA8zp8d4l3ca/AnylGsuS9Fa//nU2/MAH8q1D\nqifeoURqcA8+CO95D+yyS96VSPXDcJMaWEpZuB19dN6VSPXFcJMa2NNPw+rVhpvUk+EmNbCu822G\nm/RWhpvUwB58EMaNg733zrsSqb4YblID6zrf5v0kpbcy3KQG9dJL8MwzHpKUyjHcpAb14IPZ0HCT\n3s5wkxrU3LkwciQcfHDelUj1x3CTGtT8+VmwDRuWdyVS/THcpAa0ZQs89hi0tuZdiVSfDDepAT39\nNLz2Ghxa9uFRkgw3qQG1tWVD99yk8gw3qQG1tcH228O+++ZdiVSfDDepAbW1wSGHQEtL3pVI9clw\nkxpMRwcsWOAhSWlrDDepwTz1FKxfb7hJW2O4SQ2mqzOJPSWl3hluUoNpa4N3vtMnAUhbY7hJDaat\nLdtrG+Jvr9Qrfz2kBrJ5s51JpP4w3KQG8pvfwBtvGG5SXww3qYF4ZxKpfww3qYG0tcGOO8Kee+Zd\niVTfDDepgXR1JonIuxKpvhluUoN44w1YuNBDklJ/GG5Sg1i8OOstabhJfTPcpAYxf342NNykvhlu\nUoNoa4PRo2Hy5Lwrkeqf4SY1iLa2bK/NziRS3ww3qQFs3AiLFnlIUuovw01qAE88kT3HzXCT+sdw\nkxqAdyaRBsZwkxrA/PkwbhzsvnvelUiNwXCTGoB3JpEGxnCT6tz69fDkkx6SlAaiKuEWETMi4rcR\nsSwiLiozf0RE3Fya/0hETK7GcqVmsHAhbNliuEkDUXG4RUQLcBlwMrA/8NGI2L9Hs78B/phSeg/w\nfeBblS5XahZ2JpEGrhp7btOAZSmlZ1NKm4CbgNN7tDkduK40/jNgeoRnD6T+aGuDXXaBXXfNuxKp\ncVQj3HYDXug23V56r2yblFIHsA4YU4VlS4U3f753JpEGqhrhVu5XLm1DGyLigohoi4i2VatWVaE0\nqbG99hosWZL1lJTUf9UIt3ag+9U3E4HlvbWJiKHAjsDanh+UUpqVUmpNKbWOGzeuCqVJjW3BAujs\n9HybNFDVCLd5wF4RMSUihgMzgdk92swGzi2N/wXwq5TS2/bcJL1VV2cS99ykgRla6QeklDoi4kLg\nTqAFuCal9GREfANoSynNBq4Gro+IZWR7bDMrXa7UDNraYLfdYMKEvCuRGkvF4QaQUpoDzOnx3iXd\nxjcCZ1djWVIz6XrMjaSB8Q4lUp165RV4+mnDTdoWhptUpx5/HFLyfJu0LQw3qU7ZmUTadoabVKfa\n2mDSJBg/Pu9KpMZjuEl1ys4k0rYz3KQ69PLLsGyZ4SZtK8NNqkOPPZYNDTdp2xhuUh3q6kxyyCH5\n1iE1KsNNqkNtbTBlCozx2RnSNjHcpDo0b56HJKVKGG5SnVm1Cp57Dg47LO9KpMZluEl1put8m+Em\nbTvDTaozjz6aPXXbO5NI285wk+rMvHmw774walTelUiNy3CT6khKWbh5SFKqjOEm1ZEXXoCVK2Ha\ntLwrkRqb4SbVkXnzsqF7blJlDDepjsybB8OGwfvel3clUmMz3KQ6Mm8evPe9MGJE3pVIjc1wk+pE\nZ2d2jZuHJKXKGW5SnVi6FF55xXCTqsFwk+qEnUmk6jHcpDrx6KOw/faw3355VyI1PsNNqhNz52bX\ntw0dmnclUuMz3KQ68Prr8PjjcOSReVciFYPhJtWBtjbYssVwk6rFcJPqwEMPZcPDD8+3DqkoDDep\nDsydC/vsA2PG5F2JVAyGm5SzlLI9Nw9JStVjuEk5W7oU1qwx3KRqMtyknHWdbzPcpOox3KScPfQQ\nvOtd2dO3JVWH4SblbO5cOOIIGOJvo1Q1/jpJOVqzBhYvhqOOyrsSqVgMNylHDzyQDY89Nt86pKIx\n3KQc3XdfdrPk1ta8K5GKxXCTcnT//dkhyeHD865EKpaKwi0idoqIuyNiaWk4upd2WyJiQek1u5Jl\nSkWxejUsWgTHHJN3JVLxVLrndhFwb0ppL+De0nQ5G1JKU0uv0ypcplQIXefbDDep+ioNt9OB60rj\n1wFnVPh5UtPoOt/mk7el6qs03HZOKb0EUBqO76XdyIhoi4iHI6LXAIyIC0rt2latWlVhaVJ9u/9+\nOPpoGDYs70qk4unzmb8RcQ+wS5lZFw9gOZNSSssjYg/gVxGxKKX0TM9GKaVZwCyA1tbWNIDPlxrK\nypXZ9W1/+Zd5VyIVU5/hllI6vrd5EbEiIiaklF6KiAnAyl4+Y3lp+GxE3A8cDLwt3KRmcddd2fDE\nE/OtQyqqSg9LzgbOLY2fC9zWs0FEjI6IEaXxscBRwG8qXK7U0O64A8aNg4MPzrsSqZgqDbdLgRMi\nYilwQmmaiGiNiB+X2uwHtEXEQuA+4NKUkuGmptXZCXfeCSed5P0kpcHS52HJrUkprQGml3m/Dfjb\n0vhDwEGVLEcqkscey65xO/nkvCuRisv/G6Uau+MOiIATTsi7Eqm4DDepxu64I7uX5LhxeVciFZfh\nJtXQ6tXZ89tmzMi7EqnYDDephmbPzjqUnHlm3pVIxWa4STV0yy0weTJMnZp3JVKxGW5Sjbz6Ktx9\nN5x1VtahRNLgMdykGpkzBzZt8pCkVAuGm1Qjt9wCO+8MRxyRdyVS8RluUg289hrcfjuccQa0tORd\njVR8hptUA7fcAuvXw8c/nnclUnMw3KQauP562GMPOPLIvCuRmoPhJg2yF1+Ee++Fj33MXpJSrRhu\n0iC74QZIKQs3SbVhuEmDqLMTrrwSPvAB2GuvvKuRmofhJg2iu+6CZ5+Fz34270qk5mK4SYPo8suz\na9u8cFuqLcNNGiTPPptd23b++TB8eN7VSM3FcJMGyXe+A8OGwac+lXclUvMx3KRB8NJLcM018MlP\nwm675V2N1HwMN2kQfO970NEB//APeVciNSfDTaqyF1+Eyy6DmTNhzz3zrkZqToabVGWXXJLttX3z\nm3lXIjUvw02qokWL4N//Hf7u77J7SUrKh+EmVUlnJ3z60zB6NFx8cd7VSM1taN4FSEVxxRXwv/8L\n114LO+2UdzVSc3PPTaqC3/0OLroIjj8ePvGJvKuRZLhJFdq0CT7yERgyBGbN8rE2Uj3wsKRUoS98\nAebNy562PWVK3tVIAvfcpIp8//vZNW1f/KI3R5bqieEmbaP//M9sr+3DH4ZvfSvvaiR1Z7hJ2+Cq\nq7KOI8cdl4VcS0veFUnqznCTBmDLFvjyl+GCC+Dkk7NH2owcmXdVknoy3KR+euEFOOkk+Pa3s4u1\nb70Vttsu76oklWO4SX3YvDl7ovYBB8DcuXD11dm0DyCV6pfhJvVi82a48cYs1D77WTjsMFi8GM47\nL+/KJPXF69ykHpYtyzqJzJqVPXT0wAPhttvg1FO9QFtqFBWFW0ScDXwd2A+YllJq66XdDOAHQAvw\n45TSpZUsV6qmV16Bhx+GBx7IQmzx4izEZsyAK6+EU06xN6TUaCrdc1sMnAVc2VuDiGgBLgNOANqB\neRExO6X0mwqXLfVbZyesXg3Ll8PSpbBkSfZ68sns1dmZ3T7r6KOzC7PPPBPe/e68q5a0rSoKt5TS\nEoDY+rGaacCylNKzpbY3AacDgx5ur78OGzf+aTqlt853ur6nIet6v2lT368NG2Ddumwv7JVX/jS+\ndm12aPEPf8geINolIguv/faDs87KQu3974dRo95eg6TGU4tzbrsBL3SbbgfeX4Pl8rWvwXe/W4sl\nqV684x2www6w447ZcPRo2H9/2HXX7DVhQvYQ0X32ge23z7taSYOlz3CLiHuAXcrMujildFs/llFu\nt67M/+gQERcAFwBMmjSpHx+9dWec8fZDSz13Mp2ur+me7w0ZAiNGZN3ut/YaOTLb6xpqFylJ9CPc\nUkrHV7iMdmD3btMTgeW9LGsWMAugtbW1bAAOxNFHZy9JUnOpxXVu84C9ImJKRAwHZgKza7BcSVKT\nqijcIuLMiGgHjgB+GRF3lt7fNSLmAKSUOoALgTuBJcB/pZSerKxsSZJ6V2lvyVuBW8u8vxw4pdv0\nHGBOJcuSJKm/vP2WJKlwDDdJUuEYbpKkwjHcJEmFY7hJkgrHcJMkFY7hJkkqHMNNklQ4hpskqXAM\nN0lS4RhukqTCMdwkSYVjuEmSCidSqviZoIMiIlYBz1fho8YCq6vwObXUaDVb7+BrtJobrV5ovJqb\ntd53p5TG9dWobsOtWiKiLaXUmncdA9FoNVvv4Gu0mhutXmi8mq136zwsKUkqHMNNklQ4zRBus/Iu\nYBs0Ws3WO/gareZGqxcar2br3YrCn3OTJDWfZthzkyQ1mUKEW0ScHRFPRkRnRLT2mPeViFgWEb+N\niJN6+fopEfFIRCyNiJsjYnhtKn9z+TdHxILS67mIWNBLu+ciYlGpXVsta+xRx9cj4sVuNZ/SS7sZ\npfW+LCIuqnWd3er4TkQ8FRFPRMStEfGuXtrlun77Wl8RMaK0rSwrba+Ta11jj3p2j4j7ImJJ6ffv\n/5Zpc0xErOu2rVySR63d6tnqzzgy/1pax09ExCF51Nmtnn26rbsFEfFKRHyuR5tc13FEXBMRKyNi\ncbf3doqIu0t/U++OiNG9fO25pTZLI+LcqhaWUmr4F7AfsA9wP9Da7f39gYXACGAK8AzQUubr/wuY\nWRq/Avh0jt/Ld4FLepn3HDC2Dtb314Ev9dGmpbS+9wCGl34O++dU74nA0NL4t4Bv1dv67c/6Aj4D\nXFEanwncnPN2MAE4pDQ+Cni6TM3HALfnWedAfsbAKcB/AwEcDjySd809tpE/kF3nVTfrGPggcAiw\nuNt73wYuKo1fVO53DtgJeLY0HF0aH12tugqx55ZSWpJS+m2ZWacDN6WU3kgp/Q5YBkzr3iAiAjgO\n+FnpreuAMwaz3t6UajkHuDGP5VfZNGBZSunZlNIm4Cayn0fNpZTuSil1lCYfBibmUUcf+rO+Tifb\nPiHbXqeXtplcpJReSik9Vhp/FVgC7JZXPVVyOvAfKfMw8K6ImJB3USXTgWdSStW4uUXVpJQeANb2\neLv7ttrb39STgLtTSmtTSn8E7gZmVKuuQoTbVuwGvNBtup23//KNAV7u9sevXJta+QCwIqW0tJf5\nCbgrIuZHxAU1rKucC0uHba7p5ZBDf9Z9Hs4j+8+8nDzXb3/W15ttStvrOrLtN3elQ6QHA4+UmX1E\nRCyMiP+OiANqWtjb9fUzrtftFrK99d7+8a2ndQywc0rpJcj+CQLGl2kzqOt6aLU+aLBFxD3ALmVm\nXZxSuq23LyvzXs/uof1pU7F+1v9Rtr7XdlRKaXlEjAfujoinSv81Vd3W6gV+BHyTbD19k+xQ6nk9\nP6LM1w5a19z+rN+IuBjoAG7o5WNqtn7LqJttdaAi4p3Az4HPpZRe6TH7MbLDaK+Vzs3+P2CvWtfY\nTV8/43pdx8OB04CvlJldb+u4vwZ1XTdMuKWUjt+GL2sHdu82PRFY3qPNarJDD0NL/w2Xa1OxvuqP\niKHAWcChW/mM5aXhyoi4lexQ1qD88e3v+o6Iq4Dby8zqz7qvmn6s33OBDwHTU+mAf5nPqNn6LaM/\n66urTXtpe9mRtx8OqqmIGEYWbDeklG7pOb972KWU5kTE5RExNqWUyz0R+/Ezrul2OwAnA4+llFb0\nnFFv67hkRURMSCm9VDqsu7JMm3ay84VdJpL1m6iKoh+WnA3MLPUym0L238yj3RuU/tDdB/xF6a1z\ngd72BAfT8cBTKaX2cjMj4h0RMaprnKyTxOJybQdbj3MQZ/ZSxzxgr8h6og4nO6Qyuxb19RQRM4Av\nA6ellNb30ibv9duf9TWbbPuEbHv9VW9BXQul831XA0tSSt/rpc0uXecFI2Ia2d+cNbWr8i219Odn\nPBv4RKnX5OHAuq7Daznr9ahOPa3jbrpvq739Tb0TODEiRpdObZxYeq868uphU80X2R/YduANYAVw\nZ7d5F5P1QvstcHK39+cAu5bG9yALvWXAT4EROXwP1wKf6vHersCcbjUuLL2eJDvcltf6vh5YBDxB\nthFP6FlvafoUsh50z+Rc7zKyY/sLSq+uHod1tX7LrS/gG2ShDDCytH0uK22ve+S1Tkv1HE12GOmJ\nbuv2FOBTXdsycGFpfS4k68xzZI71lv0Z96g3gMtKP4NFdOt9nWPd25OF1Y7d3qubdUwWui8Bm0t/\nh/+G7FzwvcDS0nCnUttW4Mfdvva80va8DPjratblHUokSYVT9MOSkqQmZLhJkgrHcJMkFY7hJkkq\nHMNNklQ4hpskqXAMN0lS4RhukqTC+f+tpDOLi8vhdQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1d7b4a16630>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(7, 4))\n",
    "plt.plot(x_1, y_tanh, c='blue', label='tanh')\n",
    "plt.ylim((-1.2, 1.2))\n",
    "plt.legend(loc='best')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x1d7b4b89ef0>"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAawAAAD8CAYAAAArMZDvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl0VFW+9vHvT4IEEBABFRUEveJM\ngx31KnqxHUAGQXFoRCaHy1UvCt2NDepSWc7a2IKtMmgrXsGBZhJBhhAEwbZBUNqBYIuCELUxygsy\nyJj9/rELDSGBIjXsGp7PWrVSqTqpejgp6sk5tc8+5pxDREQk1R0UOoCIiEg0VFgiIpIWVFgiIpIW\nVFgiIpIWVFgiIpIWVFgiIpIWVFgiIpIWVFgiIpIWVFgiIpIWcpL5ZPXr13dNmjRJ5lOKiEgKW7Jk\nyffOuQbRLJvUwmrSpAmLFy9O5lOKiEgKM7Ovol1WuwRFRCQtqLBERCQtqLBERCQtJPUzrPLs2LGD\noqIitm7dGjpKSsvNzeWYY46hatWqoaOIiAQRvLCKioqoVasWTZo0wcxCx0lJzjl++OEHioqKaNq0\naeg4IiJBBN8luHXrVurVq6ey2gczo169etoKFZGsFrywAJVVFLSORCTbpURhiYiI7I8K6wDMnz+f\nU089lRYtWlBYWMgrr7wS1c8dcsghCU4mIpL5VFgHYOzYsQwYMIClS5eydu3aqAtLRERiF3yUYGib\nN2/mmmuuoaioiF27dnHPPfdQv359BgwYwM6dOznzzDMZPnw4L7/8MuPGjWPmzJnMnj2bL774gsLC\nQlq0aEGvXr2oW7cukyZNYtu2baxcuZJu3bpx33337fFcc+fOZciQIUydOhWAvn37kpeXR+/evRk0\naBBTpkwhJyeHNm3aMGTIkBCrQ0QkZaVUYfXvD0uXxvcxW7SAoUMrvn/GjBkcddRRTJs2DYANGzZw\n2mmnUVBQQLNmzejZsyfDhw+nf//+LFiwgI4dO3LVVVftVT6jR49m0aJFfPLJJ9SoUYMzzzyTDh06\nkJeXt9+M69atY9KkSSxfvhwzY/369XH5t4uIZJKs3yV4+umnM3v2bAYOHMj8+fNZtWoVTZs2pVmz\nZgD06tWLd955J6rHuuSSS6hXrx7Vq1enS5cuLFiwIKqfq127Nrm5udx0001MnDiRGjVqVPrfIyKS\nqVJqC2tfW0KJ0qxZM5YsWcJbb73FnXfeSZs2bSr9WGWHnpf9Picnh5KSkp+/331cVU5ODosWLaKg\noIDXXnuNp59+mjlz5lQ6h4hIJtrvFpaZvWBm35nZJ6VuO8zM8s3s88jXuomNmTjffPMNNWrUoHv3\n7gwYMIC///3vrFq1ihUrVgDw8ssv07p1671+rlatWmzcuHGP2/Lz81m3bh0//fQTkydPplWrVnvc\nf+yxx7Js2TK2bdvGhg0bKCgoAGDTpk1s2LCB9u3bM3ToUJbGe7+oiEgGiGYLazTwNPB/pW4bBBQ4\n5x41s0GR7wfGP17iffzxx9xxxx0cdNBBVK1aleHDh7Nhwwauvvrqnwdd3HzzzXv9XPPmzcnJyeFX\nv/oVvXv3pm7dupx33nn06NGDFStW0K1bt70+v2rUqBHXXHMNzZs354QTTqBly5YAbNy4kc6dO7N1\n61acczz55JNJ+beLiKQTc87tfyGzJsBU59xpke8/Ay5wzn1rZg2Buc65E/f3OHl5ea7sCRwLCws5\n+eSTKxE9tYwePZrFixfz9NNPJ+w5MmVdiUj6Ky6GBlGdJ3jfzGyJc27/o9Oo/KCLI5xz3wJEvh5e\nyccREZE08+mncOKJMHx4cp834YMuzKwP0AegcePGiX66YHr37k3v3r1DxxARSaivvoI2bSA3F9q1\nS+5zV3YLa21kVyCRr99VtKBzbpRzLs85l9eggu3HaHZLZjutIxEJrbjYl9WWLTBzJjRpktznr2xh\nTQF6Ra73At6obIDc3Fx++OEHvSHvw+7zYeXm5oaOIiJZauNGv0W1ejVMnQqnn578DPvdJWhmrwIX\nAPXNrAi4D3gUGGdmNwKrgasrG+CYY46hqKiI4uLiyj5EVth9xmERkWTbtg2uuMLPRDR5MpQ5Yidp\n9ltYzrlrK7jrongEqFq1qs6iKyKSonbtgu7doaAAXnoJOnYMlyXrp2YSEZHyOQd9+8L48TBkCPTs\nGTaPCktERMo1eDCMGAEDB8If/hA6jQpLRETK8fTTcP/9cMMN8MgjodN4KiwREdnDq6/C7bdD584w\nciSUmcc7GBWWiIj8bOZM/1nV+ef74spJoXN6qLBERASAhQvhyivh1FNhyhSoXj10oj2psEREhMJC\n6NABjjgCZsyAOnVCJ9qbCktEJMutWeOnXMrJgVmz4MgjQycqXwrtnRQRkWT74QdfVj/+CPPmwfHH\nh05UMRWWiEiW2rTJ7wZcudIPtmjRInSifVNhiYhkoe3b4aqr4P33YcIEaN06dKL9U2GJiGSZkhLo\n3dtvVT3/PFx+eehE0dGgCxGRLOIc9Ovnj7F69FG48cbQiaKnwhIRySIPPeSnXfr97+GPfwyd5sCo\nsEREssSIEXDPPdCjB/zpT6kz5VK0VFgiIllg/Hi49VY/KvCvf4WD0vDdPw0ji4jIgSgogOuug3PO\ngXHjoGrV0IkqR4UlIpLBlizxowCbNYOpU6FGjdCJKk+FJSKSof71L2jXDurV8/MD1q0bOlFsVFgi\nIhno66/9lEvg5wc8+uiweeJBBw6LiGSYdeugbVs/T+DcuX53YCZQYYmIZJAtW+Cyy+Dzz2H6dPj1\nr0Mnih8VlohIhtixA66+Gt57z48GvPDC0IniS4UlIpIBSkrghhvgrbf8AcJXXRU6Ufxp0IWISJpz\nDgYMgDFj4IEH4H/+J3SixFBhiYikuccegyefhNtug7vvDp0mcVRYIiJp7Pnn4c474dprYejQ9Jsf\n8ECosERE0tTkyX73X9u2MHp0es4PeCAy/J8nIpKZ5s2Drl3hzDP9GYMPPjh0osRTYYmIpJkPP4RO\nneC442DaNKhZM3Si5IipsMzsd2b2qZl9YmavmlluvIKJiMjeVqyASy+FOnX8Ke7r1QudKHkqXVhm\ndjRwO5DnnDsNqAJ0jVcwERHZ07ff+s+rdu3y8wM2ahQ6UXLFeuBwDlDdzHYANYBvYo8kIiJlrV/v\nZ15fuxbmzIGTTgqdKPkqvYXlnPsaGAKsBr4FNjjnZsUrmIiIeD/95D+zWrYMJk6Es84KnSiMWHYJ\n1gU6A02Bo4CaZta9nOX6mNliM1tcXFxc+aQiIllo504/GnDBAnj55V9OGZKNYhl0cTGw0jlX7Jzb\nAUwEzi27kHNulHMuzzmX16BBgxieTkQkuzgHffrAlCnw1FPw29+GThRWLIW1GvhPM6thZgZcBBTG\nJ5aIiAwaBC++CPfeC337hk4TXiyfYS0ExgMfAB9HHmtUnHKJiGS1IUPg8cfhlltg8ODQaVJDTKME\nnXP3AffFKYuIiAAvvQR33OHPbfWXv2T2/IAHQjNdiIikkKlT4cYb4aKL/CCLKlVCJ0odKiwRkRSx\nYIHfqmrZEiZNgmrVQidKLSosEZEU8NFH0LEjNG7szxpcq1boRKlHhSUiEtjKlX5+wJo1/ZRLOgKo\nfLFOzSQiIjH47jt/MPDWrTB/Phx7bOhEqUuFJSISyI8/+i2rr7+G2bPh1FNDJ0ptKiwRkQC2boXL\nL4ePP/YzWZy71zxBUpYKS0QkyXbtguuug7ff9kPX27ULnSg9aNCFiEgSOednr5g4EZ58ErrvNWW4\nVESFJSKSRPfcA889B3fdBf37h06TXlRYIiJJMmwYPPQQ3HQTPPhg6DTpR4UlIpIEY8f6LaorroDh\nwzU/YGWosEREEmz6dOjdGy64AF55BXI03K1SVFgiIgn03ntw5ZVw+unwxhuQmxs6UfpSYYmIJMin\nn0KHDnDUUX4rq3bt0InSmwpLRCQBVq+Gtm39jOuzZsERR4ROlP60J1VEJM6Ki/38gJs2wTvvwHHH\nhU6UGVRYIiJxtHEjtG8PX33lt6yaNw+dKHOosERE4mTbNujSBT780J+A8fzzQyfKLCosEZE42LUL\nevb0s66/+CJcdlnoRJlHgy5ERGLkHNx+O4wbB3/6kz/mSuJPhSUiEqP774dnn4U77oABA0KnyVwq\nLBGRGDz7LAwe7LeqHnssdJrMpsISEamkceOgb1//edVzz2l+wERTYYmIVEJ+vj+XVatW8Prrmh8w\nGVRYIiIH6P33/azrJ50Eb74J1auHTpQdVFgiIgdg+XJ/SvsGDWDmTDj00NCJsocKS0QkSkVFfsql\nKlX8LsGGDUMnyi7a6yoiEoUffvBltX49zJsH//EfoRNlHxWWiMh+bN4MHTvCF1/43YAtW4ZOlJ1i\n2iVoZoea2XgzW25mhWZ2TryCiYikgu3b4aqrYNEieO01f9ZgCSPWLaxhwAzn3FVmdjBQIw6ZRERS\nQkkJXH89zJjhj7O64orQibJbpQvLzGoD/wX0BnDObQe2xyeWiEhYzsHvfgevvAIPPww33RQ6kcSy\nS/A4oBh40cw+NLPnzaxm2YXMrI+ZLTazxcXFxTE8nYhI8jzyCDz1FPTvD4MGhU4jEFth5QBnAMOd\ncy2BzcBev1bn3CjnXJ5zLq9BgwYxPJ2ISHKMGgV33+1nsnjiCU25lCpiKawioMg5tzDy/Xh8gYmI\npK0JE+CWW/zBwS+8AAfpaNWUUelfhXPu38AaMzsxctNFwLK4pBIRCWDOHOjWDc4+G/72N6haNXQi\nKS3WUYK3AWMjIwS/BK6PPZKISPItXgydO8MJJ8DUqVBzr0/kJbSYCss5txTIi1MWEZEgds8PWL++\nPzD4sMNCJ5LyaO+siGS1NWv8lEsHHQSzZsHRR4dOJBXR1EwikrW+/96X1YYNMHeu3x0oqUuFJSJZ\nadMm6NABVq7U/IDpQoUlIlln2zbo0gWWLIGJE6F169CJJBoqLBHJKrt2QY8e/nxWL74InTqFTiTR\n0qALEckazsH//q8/xmrIEOjdO3QiORAqLBHJGvfeCyNHwsCB8Ic/hE4jB0qFJSJZYdgwePBBuPFG\nP7GtpB8VlohkvDFj/KzrXbrAiBGazDZdqbBEJKNNm+Y/q/rNb2DsWMjRULO0pcISkYy1YIE/vX2L\nFjB5MuTmhk4ksVBhiUhG+ugj6NgRGjeG6dOhdu3QiSRWKiwRyThffglt28Ihh/jjrXTu2Mygvbki\nklH+/W+45BLYvh3mz/dbWJIZVFgikjHWr/dbVmvXQkEBnHJK6EQSTyosEckIW7bAZZdBYaE/AePZ\nZ4dOJPGmwhKRtLdjB/z2t/Duu/Daa/6UIZJ5VFgiktZKSuCGG/xW1fDhcM01oRNJomiUoIikLef8\nnIBjxsADD8DNN4dOJImkwhKRtPXIIzB0KPTrB3ffHTqNJJoKS0TS0siRvqS6d4c//1nzA2YDFZaI\npJ3x4+GWW/wp7l94AQ7SO1lW0K9ZRNJKfj506wbnngvjxkHVqqETSbKosEQkbSxaBFdcASedBG++\nCTVqhE4kyaTCEpG0UFgI7dvD4YfDzJlQt27oRJJsKiwRSXlffeUPBs7J8bsEGzYMnUhC0IHDIpLS\n1q6Fiy+GTZtg3jw4/vjQiSQUFZaIpKzdk9l+843fsmrePHQiCUmFJSIpafNmP2x92TI/wOLcc0Mn\nktBUWCKScrZv96e2/8c/4PXX/VaWSMyDLsysipl9aGZT4xFIRLLbrl1+9ooZM2DUKF9cIhCfUYL9\ngMI4PI6IZDnn/AS2f/sbDBkCN94YOpGkkpgKy8yOAToAz8cnjohks0GD4Pnn4a67/CzsIqXFuoU1\nFPgjUFLRAmbWx8wWm9ni4uLiGJ9ORDLVo4/C44/DrbfCgw+GTiOpqNKFZWYdge+cc0v2tZxzbpRz\nLs85l9egQYPKPp2IZLARI+DOO/0cgX/5i2Zel/LFsoXVCuhkZquA14ALzWxMXFKJSNZ49VW/VdWh\nA4werZnXpWKVfmk45+50zh3jnGsCdAXmOOe6xy2ZiGS8t96Cnj3h/PP9QAvNvC77or9lRCSI+fPh\nyiv97BVvvgnVq4dOJKkuLgcOO+fmAnPj8Vgikvk++AA6doQmTfzxVrVrh04k6UBbWCKSVJ99Bpde\nCoceCrNmgcZiSbRUWCKSNKtXwyWX+FGA+fnQqFHoRJJONJegiCTFd9/5svrxR5g7F5o1C51I0o0K\nS0QSbt06X1Zr1vjdgC1ahE4k6UiFJSIJtXEjtGsHy5fD1Klw3nmhE0m6UmGJSMJs2eJHAy5ZAhMm\n+K0skcpSYYlIQmzb5o+zmj8fxo6Fzp1DJ5J0p8ISkbjbudPPCzhjhp99/dprQyeSTKBh7SISVyUl\ncP31MHEiDB2qc1pJ/KiwRCRunPMT2Y4Z408R0q9f6ESSSVRYIhIXzsEdd8DIkf5EjHfdFTqRZBoV\nlojExf33wxNPQN++8PDDOqeVxJ8KS0RiNmQIDB7sP7saNkxlJYmhwhKRmAwf7ncFXnMNPPecTsAo\niaOXlohU2ksv+UEWHTvCyy9DlSqhE0kmU2GJSKWMHet3AV58sT9b8MEHh04kmU6FJSIH7PXX/ant\nL7gA3ngDcnNDJ5JsoMISkQMyYQJcdx20auVPbV+jRuhEki1UWCIStTfegK5d4eyzYdo0qFkzdCLJ\nJiosEYnKtGlw9dVwxhkwfTrUqhU6kWQbFZaI7NfMmdClCzRv7q/Xrh06kWQjFZaI7FNBAVx+OZxy\nij9b8KGHhk4k2UqFJSIVmjsXLrsMTjgB8vPhsMNCJ5JspsISkXLNm+cPCG7aFGbPhvr1QyeSbKfC\nEpG9FBRAu3bQuLG/fvjhoROJqLBEpIxZs/yW1fHH+12CRx4ZOpGIp8ISkZ9Nnw6dOkGzZjBnjras\nJLWosEQE8LNW7B4NOGcONGgQOpHInlRYIsLkyXDllf44q4ICqFcvdCKRvVW6sMyskZm9bWaFZvap\nmfWLZzARSY7x43+ZwSI/H+rWDZ1IpHw5MfzsTuAPzrkPzKwWsMTM8p1zy+KUTUQS7NVXoUcPPzfg\n9OmawUJSW6W3sJxz3zrnPohc3wgUAkfHK5iIJNbIkX7W9fPOgxkzVFaS+uLyGZaZNQFaAgvj8Xgi\nkliPPw433wzt22siW0kfMReWmR0CTAD6O+d+LOf+Pma22MwWFxcXx/p0IhID5+Cuu2DgQH+akEmT\noHr10KlEohNTYZlZVXxZjXXOTSxvGefcKOdcnnMur4HGyYoEU1ICffvCI49Anz4wZgxUrRo6lUj0\nYhklaMBfgULn3J/jF0lE4m3HDujVC559Fu64A0aMgCpVQqcSOTCxbGG1AnoAF5rZ0silfZxyiUic\nbN7sz2U1Zgw89BA89hiYhU4lcuAqPazdObcA0MteJIUVF/vTgyxaBM88A7feGjqRSOXFchyWiKSw\nL7+ESy+FNWtgwgS44orQiURio8ISyUBLlvgh6zt3+qmWzj03dCKR2GkuQZEMM3MmtG7th6u/+67K\nSjKHCkskg4wcCR06+FPav/cenHRS6EQi8aPCEskAO3dCv35+9oq2bf3p7Rs2DJ1KJL5UWCJpbsMG\nPxLwqafgd7+DKVM0L6BkJg26EEljX3zhy+rzz2HUKPjv/w6dSCRxVFgiaSo/H6691k+5NGsW/OY3\noROJJJZ2CYqkmZISePhh/1lVw4awcKHKSrKDtrBE0sj69dCzJ7z5JnTr5ncD1qwZOpVIcqiwRNLE\nRx/5OQG/+soPsOjbV3MCSnbRLkGRFOccPP00nHUWbNkCc+fCbbeprCT7qLBEUlhxMXTq5Avqwgvh\nww+hVavQqUTCUGGJpKj8fGje3I8AHDYMpk2DI44InUokHBWWSIrZuNF/PtWmDRx2GLz/Ptx+u3YB\niqiwRFJIfj6cfro/M3C/fr6smjcPnUokNaiwRFLA+vVw001+q6paNZg/H4YOhRo1QicTSR0qLJGA\nSkrgxRehWTP/deBAWLpUAytEyqPjsEQCWbzYf1a1cCGccw7MmAFnnBE6lUjq0haWSJKtXg033OCP\nq1q1Cl56CRYsUFmJ7I+2sESS5Pvv4ZFH4Jln/MHAv/893HMP1KkTOplIelBhiSRYcbGfSmnYMNi8\nGXr1gsGDoXHj0MlE0osKSyRBVq+GJ56A556Dn37y8wA+8ACcckroZCLpSYUlEkfOwbvvwvDhMG6c\nv617d/jjH+Hkk8NmE0l3KiyROPjxRxg71hfVxx/7U9T37etPWa9dfyLxocISqaRt22D6dF9UU6fC\n1q3QsqXfBXjttTpPlUi8qbBEDsCmTX76pDffhIkTYcMGaNDAz1LRoweceabm/BNJFBWWyD44B8uW\nwZw5frb0t9+G7dv9UPROneC66+CiiyBH/5NEEk7/zURK2bEDPv3Uz+U3b56/fP+9v69ZM/+51GWX\n+amTqlYNm1Uk26iwJGtt3AjLl/u5+z74AJYs8aeh37bN33/ssdC+PbRu7S/HHx82r0i2i6mwzOxS\nYBhQBXjeOfdoXFKJxIFzsG4drFnjj4laudIX1Gef+cs33/yybJ06fmqk226DX//az+137LHhsovI\n3ipdWGZWBXgGuAQoAt43synOuWXxCidSlnN+CPn33/sZJIqL97xeXAxff+1Las0a2LJlz58/9FA4\n6SS45BL/9cQT/fmmjjtOgyVEUl0sW1hnASucc18CmNlrQGdAhZUhnINdu8q/7NxZ8X2ll9m2zV+2\nbo3u+pYtvpA2btzzsvu2TZt8rvJUq+ZH7B11FJx2mt+d16iRPw6qUSNo0sTfr2ISSU+xFNbRwJpS\n3xcBZ8cWZ98+/xw6d/bXS79pJfJ6sp4nFbKUlOxZOBUVQyIcdJAvnJo1oVatXy716kHTpnveVqeO\nL5769ff8WrOmykgkk8VSWOW9Nez1FmdmfYA+AI1jPOQ/N9f/5fzLYyfnerKeJ3QWM6hSJbpLTk50\ny1Sr5n9v1arteb3sbRoWLiL7E8vbRBHQqNT3xwDflF3IOTcKGAWQl5cX09/sjRr9Mj+biIhkl1hO\n4Pg+cIKZNTWzg4GuwJT4xBIREdlTpbewnHM7zawvMBM/rP0F59yncUsmIiJSirkkfrJuZsXAV3F4\nqPrA93F4nGRR3sRLt8zplhfSL7PyJl48Mh/rnGsQzYJJLax4MbPFzrm80DmipbyJl26Z0y0vpF9m\n5U28ZGeO5TMsERGRpFFhiYhIWkjXwhoVOsABUt7ES7fM6ZYX0i+z8iZeUjOn5WdYIiKSfdJ1C0tE\nRLJMShaWmV1tZp+aWYmZ5ZW5704zW2Fmn5lZ2wp+vqmZLTSzz83s9ciBzUkTec6lkcsqM1tawXKr\nzOzjyHKLk5mxTI7BZvZ1qcztK1ju0sh6X2Fmg5Kds0yWP5nZcjP7yMwmmdmhFSwXdB3vb52ZWbXI\n62VF5DXbJNkZS2VpZGZvm1lh5P9fv3KWucDMNpR6rdwbImuZTPv8HZv3VGQdf2RmZ4TIGclyYql1\nt9TMfjSz/mWWCb6OzewFM/vOzD4pddthZpYfeV/NN7O6Ffxsr8gyn5tZr7gGc86l3AU4GTgRmAvk\nlbr9FOCfQDWgKfAFUKWcnx8HdI1cHwHcEvDf8gRwbwX3rQLqp8D6HgwM2M8yVSLr+zjg4Mjv4ZSA\nmdsAOZHrjwGPpdo6jmadAbcCIyLXuwKvB1ynDYEzItdrAf8qJ+8FwNRQGSvzOwbaA9Px85/+J7Aw\ndOZSr49/449DSql1DPwXcAbwSanbHgcGRa4PKu//HHAY8GXka93I9brxypWSW1jOuULn3Gfl3NUZ\neM05t805txJYgT/Nyc/MzIALgfGRm14CLk9k3opEslwDvBri+ePs59PJOOe2A7tPJxOEc26Wc25n\n5Nt/4OeyTDXRrLPO+Nco+NfsRZHXTdI55751zn0Qub4RKMSflSHddQb+z3n/AA41s4ahQwEXAV84\n5+IxmUJcOefeAdaVubn0a7Wi99W2QL5zbp1z7v8B+cCl8cqVkoW1D+Wd0qTsf6h6wPpSb2blLZMs\n5wNrnXOfV3C/A2aZ2ZLIrPYh9Y3sLnmhgk39aNZ9KDfg/4IuT8h1HM06+3mZyGt2A/41HFRk12RL\nYGE5d59jZv80s+lmdmpSg5Vvf7/jVH3tdqXiP2ZTbR0DHOGc+xb8HzfA4eUsk9B1HeykDmY2Gziy\nnLvuds69UdGPlXNb2WGOUZ32JFZR5r+WfW9dtXLOfWNmhwP5ZrY88pdN3O0rLzAceAC/nh7A78a8\noexDlPOzCR1iGs06NrO7gZ3A2AoeJmnruBwp83o9EGZ2CDAB6O+c+7HM3R/gd2FtinzWORk4IdkZ\ny9jf7zgV1/HBQCfgznLuTsV1HK2ErutgheWcu7gSPxbNKU2+x2/y50T+Yi33tCex2l9+M8sBugC/\n3sdjfBP5+p2ZTcLvQkrIm2m069vMngOmlnNXVKeTiaco1nEvoCNwkYvsQC/nMZK2jssRzTrbvUxR\n5DVTh713xSSNmVXFl9VY59zEsveXLjDn3Ftm9qyZ1XfOBZsDL4rfcdJfu1FoB3zgnFtb9o5UXMcR\na82soXPu28gu1e/KWaYI/xncbsfgxyLERbrtEpwCdI2MrGqK/6tjUekFIm9cbwNXRW7qBVS0xZZI\nFwPLnXNF5d1pZjXNrNbu6/hBBJ+Ut2yildmff0UFOVLqdDJmdikwEOjknNtSwTKh13E062wK/jUK\n/jU7p6LyTbTIZ2d/BQqdc3+uYJkjd3/GZmZn4d9Dfkheyr3yRPM7ngL0jIwW/E9gw+5dWwFVuPcl\n1dZxKaVfqxW9r84E2phZ3chHC20it8VHyJEoFV3wb5pFwDZgLTCz1H1340defQa0K3X7W8BRkevH\n4YtsBfA3oFqAf8No4OYytx0FvFUq4z8jl0/xu7lCre+XgY+Bj/AvyoZl80a+b48fOfZFyLyRLCvw\n+8qXRi67R9ql1Doub50B9+OLFiA38hpdEXnNHhdwnZ6H333zUan12h64efdrGegbWZf/xA92OTfw\n66Dc33GZzAY8E/kdfEypkceBMtfAF1CdUrel1DrGl+m3wI7Ie/GN+M9WC4DPI18PiyybBzxf6mdv\niLyeVwDXxzOXZroQEZG0kG7XlpyWAAAANklEQVS7BEVEJEupsEREJC2osEREJC2osEREJC2osERE\nJC2osEREJC2osEREJC2osEREJC38f/OAhoZmeriAAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1d7b4abd4e0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(7, 4))\n",
    "plt.plot(x_1, y_softplus, c='blue', label='softplus')\n",
    "plt.ylim((-0.2, 11))\n",
    "plt.legend(loc='best')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def prep_data():\n",
    "    train_X = np.asarray([13.3,14.4,15.5,16.71,16.93,14.168,19.779,16.182,\n",
    "                          17.59,12.167,17.042,10.791,15.313,17.997,15.654,\n",
    "                          19.27,13.1])\n",
    "    train_Y = np.asarray([11.7,12.76,12.09,13.19,11.694,11.573,13.366,12.596,\n",
    "                          12.53,11.221,12.827,13.465,11.65,12.904,12.42,12.94,\n",
    "                          11.3])\n",
    "    dtype = torch.FloatTensor\n",
    "    X = Variable(torch.from_numpy(train_X).type(dtype),\n",
    "                 requires_grad=False).view(17,1)\n",
    "    y = Variable(torch.from_numpy(train_Y).type(dtype),requires_grad=False)\n",
    "    return X,y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# get dynamic parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def set_weights():\n",
    "    w = Variable(torch.randn(1),requires_grad = True)\n",
    "    b = Variable(torch.randn(1),requires_grad=True)\n",
    "    return w,b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#deploy neural network model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def build_network(x):\n",
    "    y_pred = torch.matmul(x,w)+b\n",
    "    return y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Linear(in_features=17, out_features=1, bias=True)"
      ]
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#implement in PyTorch\n",
    "import torch.nn as nn\n",
    "f = nn.Linear(17,1) # Much simpler.\n",
    "f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#calculate the loss function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def loss_calc(y,y_pred):\n",
    "    loss = (y_pred-y).pow(2).sum()\n",
    "    for param in [w,b]:\n",
    "        if not param.grad is None: param.grad.data.zero_()\n",
    "    loss.backward()\n",
    "    return loss.data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# optimizing results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def optimize(learning_rate):\n",
    "    w.data -= learning_rate * w.grad.data\n",
    "    b.data -= learning_rate * b.grad.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learning_rate = 1e-4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Mafoi\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:6: UserWarning: invalid index of a 0-dim tensor. This will be an error in PyTorch 0.5. Use tensor.item() to convert a 0-dim tensor to a Python number\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(573.6302)\n",
      "tensor(38.7160)\n",
      "tensor(34.2038)\n",
      "tensor(30.3315)\n",
      "tensor(27.0084)\n"
     ]
    }
   ],
   "source": [
    "x,y = prep_data()  # x - training data,y - target variables\n",
    "w,b = set_weights() # w,b - parameters\n",
    "for i in range(5000):\n",
    "    y_pred = build_network(x) # function which computes wx + b\n",
    "    loss = loss_calc(y,y_pred) # error calculation\n",
    "    if i % 1000 == 0: \n",
    "        print(loss)\n",
    "    optimize(learning_rate)    # minimize the loss w.r.t. w, b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1d7b4da98d0>]"
      ]
     },
     "execution_count": 198,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAGjNJREFUeJzt3XmYVNWd//H3VzRIdAZEIAFC20YR\nNWhAW7MZxS0wxgVRM+OeyC+MifsoKm600cQFl0SNmWBAxBgeHSHiaCauETTjEsQFHGJQRGRRUBY3\nkO37+6Maq291N11dfeueurc+r+fJQ59Tt+t+cxs/nD517rnm7oiISPptEboAERGJhwJdRCQjFOgi\nIhmhQBcRyQgFuohIRijQRUQyQoEuIpIRCnQRkYxQoIuIZMSWSZ6sW7duXltbm+QpRURS78UXX3zf\n3bu3dlyigV5bW8uMGTOSPKWISOqZ2dvFHKcpFxGRjFCgi4hkhAJdRCQjFOgiIhmhQBcRyQgFuohI\nRijQRUQyQoEuIlJO778B08bAhnVlP1WiNxaJiFQNd/ivU+H/pubaX/9X6FJT1lO2OkI3s/FmttTM\nZjfqqzezRWb2csP/DitrlSIiabL4JbiySz7Mjx5b9jCH4kboE4DbgIkF/Te7+w2xVyQiklYbN8Kd\nQ+Cd53PtbXrAebNhy46JnL7VQHf36WZWW/5SRERSbN5TMPGofPvEydD3kERLaM8c+plmdgowAzjf\n3VfEVJOISHpsWAe37AWrFuTaX94TRjwFW3RIvJRSV7n8BtgJGAAsAW5s6UAzG2FmM8xsxrJly0o8\nnYhIBXrtj3BVt3yYD38cTn86SJhDiSN0d39v09dmdgfw0GaOHQuMBairq/NSziciUlHWfgLX7gAb\nG5Yi9h0MJ9wLZkHLKinQzaynuy9paB4NzN7c8SIimfG338HD5+fbP30eeuwarp5GWg10M5sEDAK6\nmdlCYDQwyMwGAA7MB/69jDWKiIT36XK4fsd8e+8fwhG/ClZOc4pZ5XJ8M93jylCLiEhleupaeOqa\nfPvc2dClT7h6WqA7RUVEWrJoJtxxYL59wEVw4CXh6mmFAl1EpDn1naPtkfNgm+3D1FIkbc4lItLY\nnP+Ohnn3XaF+VcWHOWiELiKS457bf6Wx8/8B//SlMPWUQCN0EZHnfxsN810Pz43KUxTmoBG6iFSz\nDetyd3o2NmoRdNw2TD3tpBG6iFSnRy6Nhvm3z86NylMa5qARuohUm88+hmt6R/su/wA6pD8O0///\nQESkWJNOgNcfzrcPuwH2/XG4emKmQBeR7PvoPbhxl2jf6JXBN9OKmwJdRLLtloGwfF6+/W+TYNds\nPjVTgS4i2bTsdfj1vtG++lVhakmIAl1Esqfwtv3/9wR8pS5MLQlSoItIdrz9v3Dnv+TbW2wFV7wf\nrp6EKdBFJBsKR+VnzYTtdwpTSyC6sUhE0m32lGiY9/x6bq68ysIcNEIXkbRqbjOtFGxxW04aoYtI\n+vzvrdEw739sara4LSeN0EUkPdavhau7R/suWQJf+GKYeiqMRugikg4PXxAN8/1H5kblCvPPaYQu\nIpVtzYdwbcEDma9YDlt0CFNPBVOgi0jluvtoePPJfPvIW2GvU8LVU+FaDXQzGw8cDix19/4Fr10A\njAG6u3v1rN4XkfJatQhu3j3al8HNtOJWzAh9AnAbMLFxp5n1AQ4FFsRflohUrRt3g48W59sn3g99\nDw1XT4q0GujuPt3Mapt56WbgQmBqzDWJSDV67zX4zbejfRnfTCtuJc2hm9mRwCJ3f8Va+RXIzEYA\nIwBqampKOZ2IZF3hbfsjpkGvAWFqSbE2B7qZfRG4FPheMce7+1hgLEBdXZ239XwikmHzpsHEI/Pt\njv8Mo94JV0/KlTJC3wnYEdg0Ov8KMNPM9nX3d+MsTkQyrHBUfs6rsN0OYWrJiDYHurvPAnpsapvZ\nfKBOq1xEpCjTb4Anr8q3+3wDhj8arp4MKWbZ4iRgENDNzBYCo919XLkLE5GM2bgBftY12nfhW/DF\nrs0fL21WzCqX41t5vTa2akQkm35/DLzxeL69dRe4+O1w9WSU7hQVkfJZ+wn8ole0b9Qi6LhtmHoy\nToEuIuUxZmf4ZFm+/dVBcIpuWyknBbqIxOujd+HGftE+baaVCAW6iMSncCnit86EwT8PU0sVUqCL\nSPu9Owv+c79on27bT5wCXUTap3BUfvgvoe5HYWqpcgp0ESnNS7+HqWdE+zQqD6riA/2BlxYx5pHX\nWbxyNb26dGLk4H4MHdg7dFki1a1wVH7SFNj54DC1yOcqOtAfeGkRo6bMYvW6DQAsWrmaUVNmASjU\nRUL48yXw3K+jfRqVV4yKDvQxj7z+eZhvsnrdBsY88roCXSRphaPy4Y9Bn33D1CLNquhAX7xydZv6\nRaQMxg+BBc9G+zQqL1qS08YVHei9unRiUTPh3atLpwDViFSZDevgqm7RvnNnQRc9qKZYSU8bbxH7\nO8Zo5OB+dNoqendZp606MHJwvxa+Q0RiUd+5aZjXr1KYt9Hmpo3LoaJH6Jv+BdMqF5GErF4B19VG\n+7SZVsmSnjau6ECHXKgrwEUSUPihZ6eucNFbYWrJiKSnjSt6ykVEEvD+3KZhfsUKhXkMkp42rvgR\nuohsXrtWURQGeb/vw/F/iL/IKpX0tLECXSTFSl5F8eZf4O6h0T4tRSyLJKeNNeUikmIlraKo7xwN\n80GXKMwzQiN0kRRr0yqKJ34GT98Y7VOQZ4oCXSTFil5FUThX/v2bYJ/hZaxMQmh1ysXMxpvZUjOb\n3ajvKjN71cxeNrNHzazX5t5DRMqj1VUUN+7WNMzrVynMM6qYOfQJwJCCvjHuvqe7DwAeAq6IuzAR\nad3Qgb25Ztge9O7SCQN6d+nENcP2YOiAXrkg/2hx/uBTH9IUS8a1OuXi7tPNrLag78NGzW0Aj7cs\nESlWk1UU9Z1hasFBCvI2S+OzGEqeQzeznwOnAKuAA2OrSERKs/ZT+EXPaN8Zf4Puu4SppxlpCcm0\nPouh5GWL7n6pu/cB7gHObOk4MxthZjPMbMayZctKPZ2IbE5956ZhXr+q4sJ81JRZLFq5Gicfkg+8\ntCh0aU0kvalWXOJYh/4H4JiWXnT3se5e5+513bt3j+F0IvK5FW83/dDz4gUVOcWSppBM67MYSppy\nMbO+7j63oXkk8Pf4ShKRohQGObQa5CGnPNIUkml9FkMxyxYnAc8C/cxsoZkNB641s9lm9irwPeCc\nMtcpIpvMm9b8ZlpFhHnIKY+WwrASQzKtz2IoZpXL8c10jytDLVLF0vJhWXCFQd7hC3B5cZ9NhX5G\n78jB/SIfNELlhmRan8WgO0UluLSuKEjUs7+GRy6J9rVxnjz0lEfaQjKNz2JQoEtwoUeOFS+mLW4r\nYV44jSGZJtptUYILPXKsWPee3Pxt+yXuV57WeWEpnkboElwljBwrTmGQH3oVfOfsdr1l2qY8pO0U\n6BJcmj4si1OzHwT/aR9Y90n0wBjXlGvKI9sU6BJcNY4cCz8IXrzyE4ZO3T160MkPwE7aVUOKp0CX\nilBtI8fGHwTP3/qEpgdU4J2ejWmZaWXSh6IiASxeuZpOrGkS5t/97JepCPO07MlSbRToIgG8tfUJ\nzNn6tEhf7Zo/sLHzDoEqKl6a9mSpNppyEWmjdk03rFwAv9wj0tVvzQQ+4wup+SBYy0wrlwJdpA3a\ndVdrM5tpfWfrP7J2zWp6p2geWstMK5cCXaQNSrqrdf5fYcJh0b7RK8GMv5apznKq1mWmaaBAF2mD\nNk83FI7Kd/gO/OhPMVeVrGpcZpoWCnSRNih6umHGeHjovGhfha9eaYtqW2aaFlrlItIGRe2HUt85\nGub7nZepMJfKpRG6SBtsdrrhwbNg5sToNyjIJUEKdJE2ana6oXCu/Ng7of+w5IoSQYEu0j637g0f\nvBHt06hcAlGgi5Ri4wb4Wddo34hp0GtAmHpEUKCLbFazd4U+Pgg+WRo9UKNyqQAKdJEWFN4VumLl\niqZb3F4wF7btEaA6kaZaDXQzGw8cDix19/4NfWOAI4C1wJvAj9x9ZTkLFUla2re4lepTzDr0CcCQ\ngr7HgP7uvifwD2BUzHWJBLd45Wp68X6TMO+7ZqLCXCpSqyN0d59uZrUFfY82aj4HHBtvWSLhvVUQ\n5C9u7Msxa6+ktzahkgoVxxz6acC9MbyPSGV4528w7pBIV+2aewDTJlRS0doV6GZ2KbAeuGczx4wA\nRgDU1NS053Qi5Vdwg9CbXz2ZUxYPw9ZoEyqpfCUHupmdSu7D0oPd3Vs6zt3HAmMB6urqWjxOJKhX\n74MpP4721a9iJ0jlFrdSnUoKdDMbAlwEHODun8ZbkkjCCm/bP+IW2PvUMLWItEMxyxYnAYOAbma2\nEBhNblVLR+AxMwN4zt1PL2OdIvF7/Ep45qZon1avSIoVs8rl+Ga6x5WhFpHkFI7Kf/gw1O4XphaR\nmOhOUakuE4fCvL9E+zQql4xQoFehdj21Pq02rIerto/2nf0ydN0xTD0JqcqfdRVToFeZdj21Pq1+\n3hPWFXx232hUntXQq8qfdZXTI+iqzOaeWp85a1bl5sobh/nFC5qE+agps1i0cjVOPvQeeGlR8vXG\nrKp+1gJohF512vzU+rQq/NBzy63hsveaHLa50Ev7KLZqftbyOQV6lSn6qfVlkMjUxvJ5cMvAaN/l\nH0CH5v+qZzn0Qv6sJQxNuVSZop5aXwaJTG3Ud46G+VcPzE2vtBDm0HK4ZSH0Qv2sJRwFepUZOrA3\n1wzbg95dOmFA7y6duGbYHmWfXijrfO78Z5pOsdSvglMeaPVbsxx6oX7WEo6mXKpQs0+tL7OyTW0U\nBvl+/wGHjC762zddhyyucoEwP2sJR4EuiYh9PnfmRHjwrGhfiTcIKfQkKzTlIomIdWqjvnM0zIfd\nobs9RdAIXRISy9TG/1wMz/8m2qcgF/mcAl0SU/LUhjtc2SXaN/xx6LNPPIWJZIQCPaMyczv77w6F\nhS9E+zQqF2mWAj2DMrGHx4Z1cFW3aN+5s6FLnzD1iKSAAr2MQo2SU387e+FSRNCoXKQICvQyCTlK\nTu3t7J8uh+sLtrMdtQg6bhumHpGU0bLFMgm5010qb2ev7xwN82165EblCnORoinQyyTkKDlVt7Mv\n+0fTKZYrVsDIuWHqEUkxTbmUScid7lJzO3thkO96OPzbPWFqEckABXqZjBzcLzKHDsmOkiv6dvY3\nnoDfD4v26UNPkXZToJdJakbJSSsclR94KRxwYZhaRDKm1UA3s/HA4cBSd+/f0HccUA/sBuzr7jPK\nWWRaVfQoOWlz/hvuPSnap1G5SKyKGaFPAG4DJjbqmw0MA35bhpokawpH5T+YCLsfFaYWkQxrNdDd\nfbqZ1Rb0zQEws/JUJdnw11vgscujfRqVi5RN2efQzWwEMAKgpqam3KeTStDcZlpnvgjddg5Tj0iV\nKPs6dHcf6+517l7XvXv3cp9OQpt6RtMwr1+lMBdJgFa5SDzWr4WrC/7BHjkPttk+TD0iVUiBLu33\n2/1hySv5dvdd4Yznw9UjUqWKWbY4CRgEdDOzhcBoYDlwK9AdeNjMXnb3weUsVCrQ6hVwXW2077Kl\nsGXHIOWIVLtiVrkc38JLf4y5FkmTwqWIexwHx/wuTC0iAmjKRdpq+Vtwy4Bo3+iVoCWsIsEp0KV4\nhaPygy6D/UeGqUVEmlCgS+veeQHGHRrt0w1CIhVHgS6bVzgqP/ZO6D+s+WNFJCgFujRv1v0weXi0\nT6NykYqmQJemCkflwx+HPvuEqUVEiqZAl7xpY+AvV0f7NCoXSQ0FujS/mdY5r8B2tUHKEZHSKNCr\n3f2nwezJ0T6NykVSSYFerdZ/Blf3iPZd9DZ06tL88SJS8RTo1eieH8DcR/LtXgNhxFOhqhGRmCjQ\nq8nqlXDdDtG+y9+HDluFqUdEYqVArxY3fQ0+XJhv738hHHRpuHpEJHYK9Kxb8Tb8as9onzbTEskk\nBXqWFd4gdNTtMPDEMLWISNkp0LNo6Ry4/ZvRPi1FFMk8BXrWFI7Kf/gnqP1OmFpEJFEK9KyY/wxM\n+H6+vWUnuOzdcPWISOIU6FlQOCo/+yXo+tUwtYhIMFuELkDaYdb90TDvNTA3V64wF6lKGqGn0caN\n8LPton0j58E224epR0QqQqsjdDMbb2ZLzWx2o76uZvaYmc1t+HO7zb2HxOiZX0bDfI8f5EblCnOR\nqlfMlMsEYEhB38XAE+7eF3iioS3ltH5tbnrl8dH5vkvfhWPuCFeTiFSUVgPd3acDywu6jwLuavj6\nLmBozHVJYw+dB1d3z7cPuCg3Kt+qU7iaRKTilDqH/iV3XwLg7kvMrEdr3yAlaG4zrSuWwxYdwtQj\nIhWt7B+KmtkIYARATU1NuU+XHXcdAW9Nz7ePvA32OjlcPSJS8UoN9PfMrGfD6LwnsLSlA919LDAW\noK6uzks8X/VYtRBu/lq0T5tpiUgRSg30B4FTgWsb/pwaW0XV7IZ+8HGjuztPmgw7HxKuHhFJlVYD\n3cwmAYOAbma2EBhNLsjvM7PhwALguHIWmXnvzob/LNhvRZtpiUgbtRro7n58Cy8dHHMt1anwtv1/\nnw49vx6mFhFJNd0pGsqbT8LdR+fbnbrCRW+Fq0dEUk+BHkLhqPzcWdBFK4BEpH20OVeSXp4UDfOa\nb+XmyhXmIhIDjdCT0NxmWhfNh07aAkdE4qMRerlNGxMN84En50blCnMRiZlG6OWyYT3c/g344I18\n32VLYcuO4WoSkUxToJfDnIfg3hPz7cHXwLd+Gq4eEakKCvQ4rVsNY3aGtR/n2jvuD6c8qNv2RSQR\nCvS4zLwbHjwz3z79GfjyHuHqEZGqo0Bvr8Itbvf8Vxg2Nlw9IlK1FOjt8fRN8MSV+fbZL0PXHcPV\nIyJVTYFeig+XwE275tvfPhu+d1W4ekREUKC33Z9HwXO359sXzIVt9cAmEQlPgV6sD96EW/fKt793\nNXz7rHD1iIgUUKC3xh3uPw1em5Lvu3gBbN255e8REQlAgb45i1+GsQfk20N/AwNOCFePiMhmKNCb\ns3EjTDgMFjyba3fqCv8xB7baOmxdIiKboUAv9NZ0uOuIfPuE+2CXweHqEREpkgJ9kw3r4Na9YeXb\nuXaPr8HpT8MWHcLWJSJSJAU6wP9NhftOybdPewRqvhmuHhGRElR3oK/9FK6rhQ2f5do7HQwnTdZm\nWiKSSu16wIWZnWNms83sNTM7N66iEjHjTvhFz3yY/+RZOHmKwlxEUqvkEbqZ9Qd+DOwLrAX+bGYP\nu/vcuIori0+Xw/WN9lsZeBIc9etw9YiIxKQ9I/TdgOfc/VN3Xw9MA46Op6wymXZ9NMzPeVVhLiKZ\n0Z459NnAz81se2A1cBgwI5aq4vbhYrhpt3z7u+fDwVeEq0dEpAxKDnR3n2Nm1wGPAR8DrwDrC48z\nsxHACICamppST1e6hy+Av92Rb498E7bplnwdIiJlZu4ezxuZ/QJY6O63t3RMXV2dz5iR0CD+/blw\nW12+PeRa+OZPkjm3iEiMzOxFd69r7bh2LVs0sx7uvtTMaoBhwLfa836xcId7T4K/P5TvG7UQOv5T\nuJpERBLQ3nXokxvm0NcBZ7j7ihhqKt2iF+GOg/LtYb+DPY8LV4+ISILaFeju/t24CmmXjRth3CG5\nQAfY9stw7quwZcewdYmIJCj9d4q++STc3Wi15ImToe8h4eoREQkkvYG+fi3cMgA+XJRr9xwAP35S\nm2mJSNVKZ6DPnpx7itAmwx+HPvuEq0dEpAKkK9A/+xiu7QO+Mdfe5V/g+Enaf0VEhDQF+gt3wJ8u\nyLfPeAG69wtXj4hIhUlHoM+cmA/zvX8IR/wqaDkiIpUoHYHeY3fo8w04djx0/kroakREKlI6Av0r\ndTD80dBViIhUtHY94EJERCqHAl1EJCMU6CIiGaFAFxHJCAW6iEhGKNBFRDJCgS4ikhEKdBGRjIjt\nmaJFncxsGfB2YidMXjfg/dBFVBBdjyhdjzxdi6jWrscO7t69tTdJNNCzzsxmFPMg12qh6xGl65Gn\naxEV1/XQlIuISEYo0EVEMkKBHq+xoQuoMLoeUboeeboWUbFcD82hi4hkhEboIiIZoUAvkZmNN7Ol\nZja7Ud8YM/u7mb1qZn80sy4ha0xSc9ej0WsXmJmbWbcQtSWtpWthZmeZ2etm9pqZXR+qvqS18N/K\nADN7zsxeNrMZZrZvyBqTYmZ9zOwvZjan4e/BOQ39Xc3sMTOb2/DndqW8vwK9dBOAIQV9jwH93X1P\n4B/AqKSLCmgCTa8HZtYHOBRYkHRBAU2g4FqY2YHAUcCe7v414IYAdYUygaZ/N64HrnT3AcAVDe1q\nsB443913A74JnGFmuwMXA0+4e1/giYZ2mynQS+Tu04HlBX2Puvv6huZzQNU8L6+569HgZuBCoGo+\nrGnhWvwEuNbdP2s4ZmnihQXSwvVw4J8bvu4MLE60qEDcfYm7z2z4+iNgDtCb3D/2dzUcdhcwtJT3\nV6CXz2nA/4QuIiQzOxJY5O6vhK6lAuwCfNfMnjezaWa2T+iCAjsXGGNm75D7baWafpsFwMxqgYHA\n88CX3H0J5EIf6FHKeyrQy8DMLiX3q9U9oWsJxcy+CFxK7tdpyT2/dztyv2aPBO4zMwtbUlA/Ac5z\n9z7AecC4wPUkysy2BSYD57r7h3G9rwI9ZmZ2KnA4cKJX95rQnYAdgVfMbD656aeZZvbloFWFsxCY\n4jkvABvJ7d9RrU4FpjR8/V9AVXwoCmBmW5EL83vcfdM1eM/Meja83hMoaUpOgR4jMxsCXAQc6e6f\nhq4nJHef5e493L3W3WvJBdpe7v5u4NJCeQA4CMDMdgG+QHVvTrUYOKDh64OAuQFrSUzDb2XjgDnu\nflOjlx4k948cDX9OLen9q3sQWTozmwQMIjfKeg8YTW4esCPwQcNhz7n76UEKTFhz18PdxzV6fT5Q\n5+6ZD7EW/m7cDYwHBgBrgQvc/clQNSaphevxOvArclNRa4CfuvuLoWpMipntBzwNzCL3WxrAJeTm\n0e8DasitCDvO3ZtbZLD591egi4hkg6ZcREQyQoEuIpIRCnQRkYxQoIuIZIQCXUQkIxToIiIZoUAX\nEckIBbqISEb8f+BhFiXoLeDvAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1d7b4da9f28>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x_numpy = x.data.numpy()\n",
    "y_numpy = y.data.numpy()\n",
    "y_pred = y_pred.data.numpy()\n",
    "plt.plot(x_numpy,y_numpy,'o')\n",
    "plt.plot(x_numpy,y_pred,'-')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x = Variable(torch.ones(4, 4) * 12.5, requires_grad=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[12.5000, 12.5000, 12.5000, 12.5000],\n",
       "        [12.5000, 12.5000, 12.5000, 12.5000],\n",
       "        [12.5000, 12.5000, 12.5000, 12.5000],\n",
       "        [12.5000, 12.5000, 12.5000, 12.5000]], requires_grad=True)"
      ]
     },
     "execution_count": 220,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fn = 2 * (x * x) + 5 * x + 6\n",
    "\n",
    "# 2x^2 + 5x + 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [],
   "source": [
    "fn.backward(torch.ones(4,4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[55., 55., 55., 55.],\n",
      "        [55., 55., 55., 55.],\n",
      "        [55., 55., 55., 55.],\n",
      "        [55., 55., 55., 55.]])\n"
     ]
    }
   ],
   "source": [
    "print(x.grad)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
